From b5d1e12a94ccbf153f15b2a7ebdd249a3d4f7c9b Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 7 May 2024 16:07:39 -0700 Subject: [PATCH 001/366] update RCEMIP zenith angle --- components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl index 3ecd465f7a1..e3d4433d4a8 100644 --- a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl +++ b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -31,4 +31,4 @@ seq_flux_mct_albdif = 0.07 seq_flux_mct_albdir = 0.07 seq_flux_atmocn_minwind = 1 -constant_zenith_deg = 42.05 \ No newline at end of file +constant_zenith_deg = 42.04 \ No newline at end of file From b33fa7dacce0a31f9552649dc14d0f774ead8780 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 7 May 2024 16:08:06 -0700 Subject: [PATCH 002/366] update RCE use case comment --- components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml b/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml index c1c8e9e027a..146a9483b36 100644 --- a/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml +++ b/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml @@ -35,7 +35,7 @@ 0.0 - + From f3278f20b89fe1ed80681e0c8d2935c911e70adb Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 14 May 2024 12:43:37 -0700 Subject: [PATCH 003/366] add RCEMIP2 compsets --- .../eam/cime_config/config_compsets.xml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 39971fbb094..01e99a9a148 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -229,6 +229,50 @@ 2000_EAM%RCE-MMF2_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV + + + FRCE-MW_295dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW_300dT0p625 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW_300dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW_300dT2p5 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW_305dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + + FRCE-MW-MMF1_295dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT0p625 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT2p5 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_305dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + From b63cbcadde51a50160d1221a740d142d20ea5c0c Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 14 May 2024 12:44:11 -0700 Subject: [PATCH 004/366] add new data ocean SST patterns or RCEMIP2 --- .../docn/cime_config/config_component.xml | 10 +++- .../cime_config/namelist_definition_docn.xml | 7 ++- .../data_comps/docn/src/docn_comp_mod.F90 | 54 ++++++++++++++++++- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/components/data_comps/docn/cime_config/config_component.xml b/components/data_comps/docn/cime_config/config_component.xml index 5a30c69df6c..431d358f995 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/components/data_comps/docn/cime_config/config_component.xml @@ -13,7 +13,7 @@ This file may have ocn desc entries. --> - DOCN + DOCN null mode prescribed ocean mode slab ocean mode @@ -45,7 +45,7 @@ char - prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null + prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquap11,sst_aquap12,sst_aquap13,sst_aquap14,sst_aquap15,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null prescribed null @@ -63,6 +63,12 @@ sst_aquap8 sst_aquap9 sst_aquap10 + + sst_aquap11 + sst_aquap12 + sst_aquap13 + sst_aquap14 + sst_aquap15 sst_aquapfile sst_aquap_constant diff --git a/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/components/data_comps/docn/cime_config/namelist_definition_docn.xml index 948902e3732..a191d088d7f 100644 --- a/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -257,7 +257,7 @@ char streams shr_strdata_nml - SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL + SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAP11,SST_AQUAP12,SST_AQUAP13,SST_AQUAP14,SST_AQUAP15,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL General method that operates on the data. This is generally implemented in the data models but is set in the strdata method for @@ -323,6 +323,11 @@ SST_AQUAP8 SST_AQUAP9 SST_AQUAP10 + SST_AQUAP11 + SST_AQUAP12 + SST_AQUAP13 + SST_AQUAP14 + SST_AQUAP15 SST_AQUAPFILE SST_AQUAP_CONSTANT SOM diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index e692882c9db..002ed655b40 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -984,6 +984,7 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) integer :: i real(r8) :: tmp, tmp1, pi real(r8) :: rlon(lsize), rlat(lsize) + real(r8) :: mean_SST, delta_SST real(r8), parameter :: pio180 = SHR_CONST_PI/180._r8 @@ -1013,8 +1014,8 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) ! Control - if (sst_option < 1 .or. sst_option > 10) then - call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 10') + if (sst_option < 1 .or. sst_option > 15) then + call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 15') end if if (sst_option == 1 .or. sst_option == 6 .or. sst_option == 7 .or. sst_option == 8) then @@ -1174,6 +1175,55 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) end do end if + !------------------------------------------------------------------------------- + ! RCEMIP phase 2 - Mock-Walker + + ! MW_295dT1p25 - mean SST = 295 / dSST = 1.25 K + + if (sst_option == 11) then + mean_SST = 295 + delta_SST = 1.25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_300dT0p625 - mean SST = 300 / dSST = 0.625 K + if (sst_option == 12) then + mean_SST = 300 + delta_SST = 0.625 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_300dT1p25 - mean SST = 300 / dSST = 1.25 K + if (sst_option == 13) then + mean_SST = 300 + delta_SST = 1.25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_300dT2p5 - mean SST = 300 / dSST = 2.5 K + if (sst_option == 14) then + mean_SST = 300 + delta_SST = 2.5 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_305dT1p25 - mean SST = 305 / dSST = 1.25 K + if (sst_option == 15) then + mean_SST = 305 + delta_SST = 1.25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + end subroutine prescribed_sst end module docn_comp_mod From 3803486652580fc8f983ec5af650c4399fa21870 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 14 May 2024 12:45:03 -0700 Subject: [PATCH 005/366] add TMQS diagnostic for RCEMIP2 --- components/eam/src/physics/cam/cam_diagnostics.F90 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/components/eam/src/physics/cam/cam_diagnostics.F90 b/components/eam/src/physics/cam/cam_diagnostics.F90 index ead4f558f05..0b4abed4472 100644 --- a/components/eam/src/physics/cam/cam_diagnostics.F90 +++ b/components/eam/src/physics/cam/cam_diagnostics.F90 @@ -306,6 +306,7 @@ subroutine diag_init() call addfld ('MQ',(/ 'lev' /), 'A','kg/m2','Water vapor mass in layer') call addfld ('TMQ',horiz_only, 'A','kg/m2','Total (vertically integrated) precipitable water', & standard_name='atmosphere_mass_content_of_water_vapor') + call addfld ('TMQS',horiz_only, 'A','kg/m2','Total (vertically integrated) saturated precipitable water') call addfld ('TTQ',horiz_only, 'A', 'kg/m/s','Total (vertically integrated) vapor transport') call addfld ('TUQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) zonal water flux') call addfld ('TVQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) meridional water flux') @@ -1335,6 +1336,15 @@ subroutine diag_phys_writeout(state, psl) ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) end do call outfld ('TMQ ',ftem, pcols ,lchnk ) +! +! Mass of saturated q vertically integrated +! + call qsat(state%t(:ncol,:), state%pmid(:ncol,:), tem2(:ncol,:), ftem(:ncol,:)) + ftem(:ncol,:) = ftem(:ncol,:) * state%pdel(:ncol,:) * rga + do k=2,pver + ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) + end do + call outfld ('TMQS ',ftem, pcols ,lchnk ) ! ! Mass of vertically integrated water vapor flux ! From 5f7fbe49d5414013fd36d3f0a3ce0b653d4102a6 Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Thu, 23 May 2024 19:16:56 -0400 Subject: [PATCH 006/366] Adds a lower limit to stem allocation to avoid negative states The stem:leaf allocation variable can go negative without this limiter, which can cause negative stem carbon pools. --- components/elm/src/biogeochem/AllocationMod.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/elm/src/biogeochem/AllocationMod.F90 b/components/elm/src/biogeochem/AllocationMod.F90 index 2795fd42c8e..001ad75d42c 100644 --- a/components/elm/src/biogeochem/AllocationMod.F90 +++ b/components/elm/src/biogeochem/AllocationMod.F90 @@ -645,7 +645,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp if (stem_leaf(ivt(p)) < 0._r8) then if (stem_leaf(ivt(p)) == -1._r8) then - f3 = (2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4 + f3 = max((2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4_r8, 0.2_r8) else f3 = max((-1.0_r8*stem_leaf(ivt(p))*2.7_r8)/(1.0_r8+exp(-0.004_r8*(annsum_npp(p) - & 300.0_r8))) - 0.4_r8, 0.2_r8) @@ -2118,7 +2118,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & if (stem_leaf(ivt(p)) < 0._r8) then if (stem_leaf(ivt(p)) == -1._r8) then - f3 = (2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4 + f3 = max((2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4_r8, 0.2_r8) else f3 = max((-1.0_r8*stem_leaf(ivt(p))*2.7_r8)/(1.0_r8+exp(-0.004_r8*(annsum_npp(p) - & 300.0_r8))) - 0.4_r8, 0.2_r8) From 6160a60eef2c2fee930e2df751ed39b7591b158e Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:06:33 -0500 Subject: [PATCH 007/366] Add shr_const to analysis mode --- components/mpas-ocean/src/analysis_members/Makefile | 5 ++++- components/mpas-ocean/src/analysis_members/shr_const_mod.F | 1 + components/mpas-ocean/src/analysis_members/shr_kind_mod.F | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 components/mpas-ocean/src/analysis_members/shr_const_mod.F create mode 100644 components/mpas-ocean/src/analysis_members/shr_kind_mod.F diff --git a/components/mpas-ocean/src/analysis_members/Makefile b/components/mpas-ocean/src/analysis_members/Makefile index c07186ad9ad..9afb438b77e 100644 --- a/components/mpas-ocean/src/analysis_members/Makefile +++ b/components/mpas-ocean/src/analysis_members/Makefile @@ -2,6 +2,9 @@ OBJS = mpas_ocn_analysis_driver.o +UTILS = shr_kind_mod.o \ + shr_const_mod.o + MEMBERS = mpas_ocn_global_stats.o \ mpas_ocn_okubo_weiss.o \ mpas_ocn_layer_volume_weighted_averages.o \ @@ -35,7 +38,7 @@ MEMBERS = mpas_ocn_global_stats.o \ all: $(OBJS) -mpas_ocn_analysis_driver.o: $(MEMBERS) +mpas_ocn_analysis_driver.o: $(UTILS) $(MEMBERS) mpas_ocn_okubo_weiss.o: mpas_ocn_okubo_weiss_eigenvalues.o diff --git a/components/mpas-ocean/src/analysis_members/shr_const_mod.F b/components/mpas-ocean/src/analysis_members/shr_const_mod.F new file mode 100644 index 00000000000..41538d8af2f --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_const_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_const_mod.F90 diff --git a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F new file mode 100644 index 00000000000..44b22288cce --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_kind_mod.F90 From 2da96ef7e2350afec441612a4e1a3b7e97594420 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:15:21 -0500 Subject: [PATCH 008/366] Use pi for debug tracer --- .../src/mode_forward/mpas_ocn_time_integration_si.F | 4 ++-- .../src/mode_forward/mpas_ocn_time_integration_split.F | 4 ++-- .../src/mode_forward/mpas_ocn_time_integration_split_ab2.F | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F index 9529fbdc6ea..8e542d5ac61 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F @@ -2587,7 +2587,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2604,7 +2604,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index 653d8819813..2b35a4efae4 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -2225,7 +2225,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2242,7 +2242,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F index c6040aeba1d..0693d00a72f 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F @@ -2405,7 +2405,7 @@ subroutine ocn_time_integrator_split_ab2(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2422,7 +2422,7 @@ subroutine ocn_time_integrator_split_ab2(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & From 561ce771eed0e886437236d1682e47170fd32ce0 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:15:30 -0500 Subject: [PATCH 009/366] Use pi in cvmix --- components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F index fcede65e420..87fb36f2145 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F @@ -345,8 +345,8 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim ! specify geometry/location cvmix_variables % SeaSurfaceHeight = ssh(iCell) cvmix_variables % Coriolis = fCell(iCell) - cvmix_variables % lat = latCell(iCell) * 180.0_RKIND / 3.14_RKIND - cvmix_variables % lon = lonCell(iCell) * 180.0_RKIND / 3.14_RKIND + cvmix_variables % lat = latCell(iCell) * 180.0_RKIND / pi + cvmix_variables % lon = lonCell(iCell) * 180.0_RKIND / pi ! fill vertical position of column ! CVMix assume top of ocean is at z=0, so building all z-coordinate data based on layerThickness From 6223ff4f520f7917120e313adf74a49cfae7cd56 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:15:45 -0500 Subject: [PATCH 010/366] Use pi in gm --- components/mpas-ocean/src/shared/mpas_ocn_gm.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_gm.F b/components/mpas-ocean/src/shared/mpas_ocn_gm.F index 6f4442103fa..3db04c4d3e8 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_gm.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_gm.F @@ -484,7 +484,7 @@ subroutine ocn_GM_compute_Bolus_velocity(statePool, & ! Compute the speed of the first baroclinic mode from the Brunt-Vaisala frequency. cGMphaseSpeed(iEdge) = max(c_min, & - sum_hN/(config_GM_spatially_variable_baroclinic_mode*3.141592_RKIND)) + sum_hN/(config_GM_spatially_variable_baroclinic_mode*pi)) end do !$omp end do @@ -615,7 +615,7 @@ subroutine ocn_GM_compute_Bolus_velocity(statePool, & ! for the first-mode (m=1) baroclinic gravity wave speed in m/s ! c_m = 1/pi integral(N dz ) - c_mode1 = max(1.0E-5_RKIND, sum_hN/3.141592_RKIND) + c_mode1 = max(1.0E-5_RKIND, sum_hN/pi) ! See Hallberg (2013) https://doi.org/10.1016/j.ocemod.2013.08.007 ! just after eqn 4 the defines the deformation From 1b92637325d779f78fceda7fb25fd3b29e596e6c Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:16:07 -0500 Subject: [PATCH 011/366] Set pi in mpas_ocn_constants --- components/mpas-ocean/src/shared/mpas_ocn_constants.F | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_constants.F b/components/mpas-ocean/src/shared/mpas_ocn_constants.F index 35ada1306f8..6c702374820 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_constants.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_constants.F @@ -117,6 +117,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ !----------------------------------------------------------------------- T0_Kelvin = 273.16_RKIND ! zero point for Celsius + pi = 3.141592_RKIND ! pi rho_air = 1.2_RKIND ! ambient air density (kg/m^3) rho_sw = config_density0 ! density of salt water (kg/m^3) rho_fw = 1.0e3_RKIND ! avg. water density (kg/m^3) @@ -159,6 +160,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ #ifdef MPAS_ESM_SHR_CONST T0_Kelvin = SHR_CONST_TKFRZ ! zero point for Celsius + pi = SHR_CONST_PI ! zero point for Celsius cp_sw = SHR_CONST_CPSW ! erg/g/K cp_air = SHR_CONST_CPDAIR ! J/kg/K rho_air = SHR_CONST_RHODAIR ! kg/m^3 From 5e7322ed4f39d51429d744117be258c25f6c4006 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:16:30 -0500 Subject: [PATCH 012/366] Use pi in conservation earth area --- .../src/analysis_members/mpas_ocn_conservation_check.F | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index bdd07e799b5..fdfc219789e 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -19,6 +19,9 @@ module ocn_conservation_check + use shr_kind_mod, only: SHR_KIND_R8 + use shr_const_mod + use mpas_derived_types use mpas_pool_routines use mpas_dmpar @@ -157,10 +160,9 @@ subroutine ocn_init_conservation_check(domain, err)!{{{ !----------------------------------------------------------------- ! taking PI from SHR_CONST_PI in share/util/shr_const_mod.F90 to match coupler - !real (kind=RKIND), parameter :: piE3SM = 3.14159265358979323846_RKIND ! pi - real (kind=RKIND), parameter :: piE3SM = 3.141592653589793_RKIND + real (kind=RKIND), parameter :: piE3SM = SHR_CONST_PI ! taking earth radius from SHR_CONST_REARTH in share/util/shr_const_mod.F90 to match coupler - real (kind=RKIND), parameter :: earthRadiusE3SM = 6.371229e6_RKIND ! radius of earth, m + real (kind=RKIND), parameter :: earthRadiusE3SM = SHR_CONST_REARTH ! radius of earth, m err = 0 From ba7b48e8b2ded2735203aad3ee899c6a7aecb963 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:40:17 -0500 Subject: [PATCH 013/366] Set pi in mpas_ocn_constants --- components/mpas-ocean/src/shared/mpas_ocn_constants.F | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_constants.F b/components/mpas-ocean/src/shared/mpas_ocn_constants.F index 6c702374820..08b70dce377 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_constants.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_constants.F @@ -65,6 +65,7 @@ module ocn_constants real (kind=RKIND), public :: & T0_Kelvin ,&! zero point for Celsius + pi ,&! pi mpercm ,&! meters per m cmperm ,&! m per meter days_per_second ,&! days per second From 58a8d60ce428d6e493b596456f677ec2f694bc0e Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:47:52 -0500 Subject: [PATCH 014/366] Add shr_const to analysis mode --- components/mpas-ocean/src/analysis_members/shr_const_mod.F | 2 +- components/mpas-ocean/src/analysis_members/shr_kind_mod.F | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 120000 components/mpas-ocean/src/analysis_members/shr_const_mod.F mode change 100644 => 120000 components/mpas-ocean/src/analysis_members/shr_kind_mod.F diff --git a/components/mpas-ocean/src/analysis_members/shr_const_mod.F b/components/mpas-ocean/src/analysis_members/shr_const_mod.F deleted file mode 100644 index 41538d8af2f..00000000000 --- a/components/mpas-ocean/src/analysis_members/shr_const_mod.F +++ /dev/null @@ -1 +0,0 @@ -../../../../share/util/shr_const_mod.F90 diff --git a/components/mpas-ocean/src/analysis_members/shr_const_mod.F b/components/mpas-ocean/src/analysis_members/shr_const_mod.F new file mode 120000 index 00000000000..c471e79113f --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_const_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_const_mod.F90 \ No newline at end of file diff --git a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F deleted file mode 100644 index 44b22288cce..00000000000 --- a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F +++ /dev/null @@ -1 +0,0 @@ -../../../../share/util/shr_kind_mod.F90 diff --git a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F new file mode 120000 index 00000000000..77a61f967b6 --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_kind_mod.F90 \ No newline at end of file From 17aed5bb359eb04727c4bcc5e016fbda05422599 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 1 Jul 2024 14:25:51 -0500 Subject: [PATCH 015/366] Use MPASO's pi instead of MPAS-framework's pi in AM --- .../src/analysis_members/mpas_ocn_harmonic_analysis.F | 2 +- .../analysis_members/mpas_ocn_lagrangian_particle_tracking.F | 2 +- .../mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F | 4 ++-- .../mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F index ee1495a3e59..c60ac179ccf 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F @@ -681,7 +681,7 @@ SUBROUTINE harmonic_analysis_solve(MNP,nfreq,hmat,GLOELV,haff,haface,emagt,phase REAL(kind=RKIND),ALLOCATABLE :: hap(:),hax(:) REAL(kind=RKIND),ALLOCATABLE :: ha(:,:) - convrd=180.0_RKIND/pii + convrd=180.0_RKIND/pi mm = 2*nfreq ALLOCATE ( PHASEE(nfreq),EMAG(nfreq) ) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F index ba02542eb45..8a8a2665bcb 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F @@ -3833,7 +3833,7 @@ subroutine convert_latlon_from_xyz(lat, lon, x, y, z) !{{{ ! ensure range in 0, 2*pi if (lon < 0.0_RKIND) then - lon = 2*pii + lon + lon = 2*pi + lon end if end subroutine convert_latlon_from_xyz !}}} diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F index 88c9960092b..64ab02e0573 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F @@ -946,8 +946,8 @@ subroutine ocn_compute_eddy_stats(dminfo, block, nVertLevels, nCells, nCellsSolv ! for lat/lon coordinates, convert from radians to degrees for output if (config_AM_okuboWeiss_use_lat_lon_coords) then - wsPosX = wsPosX *180.0_RKIND/pii - wsPosY = wsPosY *180.0_RKIND/pii + wsPosX = wsPosX *180.0_RKIND/pi + wsPosY = wsPosY *180.0_RKIND/pi end if call mpas_timer_stop("stats per proc") diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F index 78adb9a0583..498f3f09c7a 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F @@ -62,7 +62,7 @@ module ocn_time_filters !-------------------------------------------------------------------- #ifdef TIME_FILTERS_DEBUG integer :: iEdgeOutput = 0, iBlockOutput = 0, iklevel = 1 - real (kind=RKIND) :: lonEdgePoint = 10.0_RKIND*pii/180.0_RKIND, latEdgePoint = 30.0_RKIND*pii/180.0_RKIND + real (kind=RKIND) :: lonEdgePoint = 10.0_RKIND*pi/180.0_RKIND, latEdgePoint = 30.0_RKIND*pi/180.0_RKIND #endif !*********************************************************************** @@ -182,7 +182,7 @@ subroutine ocn_init_time_filters(domain, err)!{{{ call mpas_pool_get_subpool(block % structs, 'mesh', statePool) call mpas_pool_get_array(statePool, 'latEdge', latEdge) call mpas_pool_get_array(statePool, 'lonEdge', lonEdge) - print *, 'lon = ', 180.0_RKIND/pii*lonEdge(iEdgeOutput), ' lat = ', 180.0_RKIND/pii*latEdge(iEdgeOutput), & + print *, 'lon = ', 180.0_RKIND/pi*lonEdge(iEdgeOutput), ' lat = ', 180.0_RKIND/pi*latEdge(iEdgeOutput), & ' iklevel=',iklevel, ' iEdgeOutput=',iEdgeOutput, ' iBlockOutput = ', iBlockOutput #endif From 5ce53c730807bfb626fd811640fb501ceed310f1 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 1 Jul 2024 14:26:44 -0500 Subject: [PATCH 016/366] Use MPASO's pi instead of MPAS-framework's pi in shared --- components/mpas-ocean/src/shared/Makefile | 2 +- .../src/shared/mpas_ocn_manufactured_solution.F | 4 ++-- components/mpas-ocean/src/shared/mpas_ocn_mesh.F | 3 ++- components/mpas-ocean/src/shared/mpas_ocn_tendency.F | 4 ++-- .../mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F | 4 ++-- .../src/shared/mpas_ocn_tracer_advection_shared.F | 10 +++++----- .../mpas_ocn_vel_forcing_topographic_wave_drag.F | 6 +++--- .../mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F | 2 +- .../src/shared/mpas_ocn_vel_self_attraction_loading.F | 6 +++--- .../src/shared/mpas_ocn_vel_tidal_potential.F | 10 +++++----- 10 files changed, 26 insertions(+), 25 deletions(-) diff --git a/components/mpas-ocean/src/shared/Makefile b/components/mpas-ocean/src/shared/Makefile index 35017551c75..d378b68624b 100644 --- a/components/mpas-ocean/src/shared/Makefile +++ b/components/mpas-ocean/src/shared/Makefile @@ -143,7 +143,7 @@ mpas_ocn_tracer_advection_std.o: mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_trac mpas_ocn_tracer_advection_vert.o: mpas_ocn_mesh.o mpas_ocn_config.o -mpas_ocn_tracer_advection_shared.o: mpas_ocn_mesh.o mpas_ocn_config.o +mpas_ocn_tracer_advection_shared.o: mpas_ocn_constants.o mpas_ocn_mesh.o mpas_ocn_config.o mpas_ocn_tracer_hmix_redi.o: mpas_ocn_constants.o mpas_ocn_config.o diff --git a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F index e9ad6786103..2631ba1737b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F @@ -217,8 +217,8 @@ subroutine ocn_manufactured_solution_init(domain, err)!{{{ if (.not. config_use_manufactured_solution) return - kx = 2.0_RKIND*pii / config_manufactured_solution_wavelength_x - ky = 2.0_RKIND*pii / config_manufactured_solution_wavelength_y + kx = 2.0_RKIND*pi / config_manufactured_solution_wavelength_x + ky = 2.0_RKIND*pi / config_manufactured_solution_wavelength_y eta0 = config_manufactured_solution_amplitude diff --git a/components/mpas-ocean/src/shared/mpas_ocn_mesh.F b/components/mpas-ocean/src/shared/mpas_ocn_mesh.F index 5e0690704e4..9eadb339b66 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_mesh.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_mesh.F @@ -33,6 +33,7 @@ module ocn_mesh use mpas_log use ocn_config + use ocn_constants implicit none private @@ -1792,7 +1793,7 @@ subroutine ocn_meshScaling() !{{{ ! neighboring cells are circles for this calculation. cellWidth = 2.0_RKIND* & sqrt((areaCell(cell1) + areaCell(cell2))/ & - 2.0_RKIND/pii) + 2.0_RKIND/pi) meshScalingDel2(iEdge) = cellWidth/ & config_hmix_ref_cell_width meshScalingDel4(iEdge) = (cellWidth/ & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index a16e52f1449..a17c4815074 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -1453,7 +1453,7 @@ subroutine ocn_tend_freq_filtered_thickness(tendPool, statePool, & do k = 1, maxLevelCell(iCell) tend_lowFreqDivergence(k,iCell) = & - -2.0*pii/thickness_filter_timescale_sec & + -2.0*pi/thickness_filter_timescale_sec & *(lowFreqDivergence(k,iCell) - div_hu(k) & + div_hu_btr * layerThickness(k,iCell)/totalThickness) @@ -1461,7 +1461,7 @@ subroutine ocn_tend_freq_filtered_thickness(tendPool, statePool, & div_hu_btr*layerThickness(k,iCell)/totalThickness + & lowFreqDivergence(k,iCell) + & use_highFreqThick_restore* & - (-2.0*pii/highFreqThick_restore_time_sec* & + (-2.0*pi/highFreqThick_restore_time_sec* & highFreqThickness(k,iCell) ) end do ! vert (k) loop diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F index 2a9b400dd90..48901fbf5d5 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F @@ -243,8 +243,8 @@ subroutine ocn_tidal_forcing_build_array(domain, meshPool, forcingPool, statePoo ! compute the tidalHeight if (trim(config_tidal_forcing_model) == 'monochromatic') then tidalHeight = config_tidal_forcing_monochromatic_amp * & - SIN(2.0_RKIND*pii/config_tidal_forcing_monochromatic_period * daysSinceStartOfSim - & - pii*config_tidal_forcing_monochromatic_phaseLag/180.0_RKIND) - & + SIN(2.0_RKIND*pi/config_tidal_forcing_monochromatic_period * daysSinceStartOfSim - & + pi*config_tidal_forcing_monochromatic_phaseLag/180.0_RKIND) - & config_tidal_forcing_monochromatic_baseline elseif (trim(config_tidal_forcing_model) == 'linear') then tidalHeight = max(config_tidal_forcing_linear_min, & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F index 2d14c961fba..3439776094d 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F @@ -24,10 +24,12 @@ module ocn_tracer_advection_shared use mpas_derived_types use mpas_hash use mpas_sort + use mpas_constants use mpas_geometry_utils use mpas_log use ocn_config + use ocn_constants use ocn_mesh implicit none @@ -370,7 +372,6 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ xec, yec, zec, &! arc bisection coords thetae_tmp, &! angle xv1, xv2, yv1, yv2, zv1, zv2, &! vertex cart coords - pii, &! pi math constant length_scale, &! length scale cos2t, costsint, sin2t ! trig function temps @@ -413,7 +414,6 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ ! Initialize derivTwo and pi - pii = 2.*asin(1.0) derivTwo(:,:,:) = 0.0_RKIND do iCell = 1, nCellsAll @@ -461,9 +461,9 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ end do if ( zc(1) == 1.0_RKIND) then - theta_abs(iCell) = pii/2.0_RKIND + theta_abs(iCell) = pi/2.0_RKIND else - theta_abs(iCell) = pii/2.0_RKIND & + theta_abs(iCell) = pi/2.0_RKIND & - mpas_sphere_angle( xc(1), yc(1), zc(1), & xc(2), yc(2), zc(2), & 0.0_RKIND, 0.0_RKIND, 1.0_RKIND) @@ -509,7 +509,7 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ angle_2d(i) = angleEdge(edgesOnCell(i,iCell)) iEdge = edgesOnCell(i,iCell) if ( iCell .ne. cellsOnEdge(1,iEdge)) & - angle_2d(i) = angle_2d(i) - pii + angle_2d(i) = angle_2d(i) - pi xp(i) = dcEdge(edgesOnCell(i,iCell)) * cos(angle_2d(i)) yp(i) = dcEdge(edgesOnCell(i,iCell)) * sin(angle_2d(i)) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F index 8f641ca4f31..a024ec97e51 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F @@ -199,7 +199,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ call mpas_log_write(" Topographic wave drag scheme is: Jayne and St. Laurent") call mpas_log_write("") tensorScheme = .false. - kappa = pii/10000.0_RKIND + kappa = pi/10000.0_RKIND do iEdge = 1, nEdgesAll kmax = maxLevelEdgeTop(iEdge) @@ -241,7 +241,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ topographicWaveDrag(iEdge) = topographicWaveDragCoeff * gam & * bed_slope_edges(iEdge)**2 * Nbar * Nb & - / (8.0_RKIND * omegaM2 * pii**2) + / (8.0_RKIND * omegaM2 * pi**2) endif enddo else if (config_topographic_wave_drag_scheme.EQ."LGF") then @@ -256,7 +256,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ topographicWaveDrag(iEdge) = topographicWaveDragCoeff & * (sqrt((topo_buoyancy_N1B(iEdge)**2 - omegaM2**2) & * (topo_buoyancy_N1V(iEdge)**2 - omegaM2**2))) & - / (4.0_RKIND*pii*omegaM2) + / (4.0_RKIND*pi*omegaM2) normalCoeffTWD(iEdge) = (lonGradEdge(iEdge)**2) & * (cos(angleEdge(iEdge))**2) + (latGradEdge(iEdge)**2)*(sin(angleEdge(iEdge))**2) & - 2.0_RKIND*latGradEdge(iEdge)*lonGradEdge(iEdge)*sin(angleEdge(iEdge))*(cos(angleEdge(iEdge))) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F index a6f9ec5c073..db3e78e9b88 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F @@ -159,7 +159,7 @@ subroutine ocn_vel_hmix_leith_tend(div, relVort, tend, err)!{{{ dcEdgeInv = 1.0_RKIND / dcEdge(iEdge) dvEdgeInv = 1.0_RKIND / dvEdge(iEdge) - visc2tmp = (leithParam*dxLeith*meshScalingDel2(iEdge)/pii)**3 + visc2tmp = (leithParam*dxLeith*meshScalingDel2(iEdge)/pi)**3 do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F index 90d1a4e4630..a1c7a65452f 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F @@ -863,8 +863,8 @@ subroutine ocn_vel_self_attraction_loading_init(domain,err)!{{{ ! Pre-compute sin and cos of latCell (co-latitude) values allocate(sinLatCell(nCellsAll), cosLatCell(nCellsAll)) do iCell = 1,nCellsAll - sinLatCell(iCell) = sin(0.5_RKIND*pii-latCell(iCell)) - cosLatCell(iCell) = cos(0.5_RKIND*pii-latCell(iCell)) + sinLatCell(iCell) = sin(0.5_RKIND*pi-latCell(iCell)) + cosLatCell(iCell) = cos(0.5_RKIND*pi-latCell(iCell)) enddo ! Calculate blocking indices @@ -2526,7 +2526,7 @@ subroutine associatedLegendrePolynomials(n, m, startIdx, endIdx, l, pmnm2, pmnm1 if (n == m) then do iCell = startIdx,endIdx - pmnm2(iCell) = sqrt(1.0_RKIND/(4.0_RKIND*pii))*sinLatCell(iCell)**m + pmnm2(iCell) = sqrt(1.0_RKIND/(4.0_RKIND*pi))*sinLatCell(iCell)**m do i = 1,m pmnm2(iCell) = pmnm2(iCell)*sqrt(real(2*i+1,RKIND)/real(2*i,RKIND)) enddo diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F index 211538ac0a7..daafa816fab 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F @@ -282,7 +282,7 @@ subroutine ocn_compute_tidal_potential_forcing(err)!{{{ !*** Compute eta by summing all constituent contributions do jCon = 1, nTidalConstituents - period = 2.0_RKIND*pii/tidalConstituentFrequency(jCon) + period = 2.0_RKIND*pi/tidalConstituentFrequency(jCon) nCycles = real(int(t/period),RKIND) targ = tidalConstituentFrequency(jCon)*(t - nCycles*period) + & tidalConstituentNodalPhase(jCon) + & @@ -583,7 +583,7 @@ subroutine tidal_constituent_factors(constituentList,nTidalConstituents,refTime, !h = 280.4661_RKIND + 0.98564736_RKIND*T; !p = 83.3535_RKIND + 0.11140353_RKIND*T; !N = 125.0445_RKIND - 0.05295377_RKIND*T; - !N = N*pii/180.0_RKIND + !N = N*pi/180.0_RKIND !! M2 !tidalConstituentAstronomical(j) = 2.0_RKIND*h - 2.0_RKIND*s @@ -611,7 +611,7 @@ subroutine tidal_constituent_factors(constituentList,nTidalConstituents,refTime, !tidalConstituentNodalAmplitude(j) = 1.009_RKIND + 0.187_RKIND*cos(N) - deg2rad = pii/180.0_RKIND + deg2rad = pi/180.0_RKIND T = adjust_angle(180.0_RKIND + real(refHour,RKIND)*(360.0_RKIND/24.0_RKIND)) do j = 1,nTidalConstituents @@ -742,8 +742,8 @@ subroutine orbit(year,julianDay,hour, & real (kind=RKIND) :: deg2rad,rad2deg real (kind=RKIND) :: NRad,pRad,IRad,nuRad,xiRad,nupRad,nup2Rad - deg2rad = pii/180.0_RKIND - rad2deg = 180.0_RKIND/pii + deg2rad = pi/180.0_RKIND + rad2deg = 180.0_RKIND/pi x = int((real(year,RKIND)-1901.0_RKIND)/4.0_RKIND) yr = real(year,RKIND) - 1900.0_RKIND From b3c8342d89983b960c3f7f8d1116c2fe451c3efd Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 1 Jul 2024 13:50:46 -0600 Subject: [PATCH 017/366] Increase precision of MPAS-O standalone pi Co-authored-by: Xylar Asay-Davis --- components/mpas-ocean/src/shared/mpas_ocn_constants.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_constants.F b/components/mpas-ocean/src/shared/mpas_ocn_constants.F index 08b70dce377..dd3214d8d4f 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_constants.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_constants.F @@ -118,7 +118,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ !----------------------------------------------------------------------- T0_Kelvin = 273.16_RKIND ! zero point for Celsius - pi = 3.141592_RKIND ! pi + pi = 3.14159265358979323846_RKIND ! pi rho_air = 1.2_RKIND ! ambient air density (kg/m^3) rho_sw = config_density0 ! density of salt water (kg/m^3) rho_fw = 1.0e3_RKIND ! avg. water density (kg/m^3) From 266a8e87a797f241dedd134a611341bc46a119cd Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 3 Jul 2024 02:35:01 -0500 Subject: [PATCH 018/366] Fix indexing of layers in high freq. output Previously, the layer above the one containing the desired depth was being selected, rather than the layer containing the depth. The default depth if no layer is found is now the deepest layer, rather than the first layer. This is because, if no layer is found, it means that even the deepest layer is shallower than the desired depth, and the only sensible default is the deepest layer. --- .../mpas_ocn_high_frequency_output.F | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F index 60a5dececcb..5b1dd6aabc0 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F @@ -175,7 +175,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ type (mpas_pool_type), pointer :: highFrequencyOutputAMPool type (mpas_pool_type), pointer :: tracersPool - integer :: iLevel, iLevelTarget, iCell, iEdge, i, cell1, cell2, k, eoe + integer :: iLevel, iCell, iEdge, i, cell1, cell2, k, eoe integer :: iLevel0100, iLevel0250, iLevel0700, iLevel2000 real (kind=RKIND) :: sumLayerThickness integer, pointer :: nVertLevels, nCells, nEdges @@ -377,37 +377,41 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ call mpas_pool_get_array(highFrequencyOutputAMPool, 'columnIntegratedSpeed', columnIntegratedSpeed) ! find vertical level that is just above the 100 m reference level - iLevel0100 = 1 - do iLevel=2,nVertLevels + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0100 = nVertLevels + do iLevel=1,nVertLevels if(refBottomDepth(iLevel) > 100.0_RKIND) then - iLevel0100 = iLevel-1 + iLevel0100 = iLevel exit endif enddo ! find vertical level that is just above the 250 m reference level - iLevel0250 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0250 = nVertLevels do iLevel=iLevel0100,nVertLevels if(refBottomDepth(iLevel) > 250.0_RKIND) then - iLevel0250 = iLevel-1 + iLevel0250 = iLevel exit endif enddo ! find vertical level that is just above the 700 m reference level - iLevel0700 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0700 = nVertLevels do iLevel=iLevel0250,nVertLevels if(refBottomDepth(iLevel) > 700.0_RKIND) then - iLevel0700 = iLevel-1 + iLevel0700 = iLevel exit endif enddo ! find vertical level that is just above the 2000 m reference level - iLevel2000 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel2000 = nVertLevels do iLevel=iLevel0700,nVertLevels if(refBottomDepth(iLevel) > 2000.0_RKIND) then - iLevel2000 = iLevel-1 + iLevel2000 = iLevel exit endif enddo From 53b636a5cbd3d5796021f4983b62d3d73d27e770 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 3 Jul 2024 14:33:44 -0700 Subject: [PATCH 019/366] update docn_comp_mod.F90 --- .../data_comps/docn/src/docn_comp_mod.F90 | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index 002ed655b40..ac127867365 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -1178,46 +1178,46 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) !------------------------------------------------------------------------------- ! RCEMIP phase 2 - Mock-Walker - ! MW_295dT1p25 - mean SST = 295 / dSST = 1.25 K + ! MW_295dT1p25 - mean SST = 295 K / dSST = 1.25 K if (sst_option == 11) then - mean_SST = 295 + mean_SST = 295 - TkFrz delta_SST = 1.25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_300dT0p625 - mean SST = 300 / dSST = 0.625 K + ! MW_300dT0p625 - mean SST = 300 K / dSST = 0.625 K if (sst_option == 12) then - mean_SST = 300 + mean_SST = 300 - TkFrz delta_SST = 0.625 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_300dT1p25 - mean SST = 300 / dSST = 1.25 K + ! MW_300dT1p25 - mean SST = 300 K / dSST = 1.25 K if (sst_option == 13) then - mean_SST = 300 + mean_SST = 300 - TkFrz delta_SST = 1.25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_300dT2p5 - mean SST = 300 / dSST = 2.5 K + ! MW_300dT2p5 - mean SST = 300 K / dSST = 2.5 K if (sst_option == 14) then - mean_SST = 300 + mean_SST = 300 - TkFrz delta_SST = 2.5 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_305dT1p25 - mean SST = 305 / dSST = 1.25 K + ! MW_305dT1p25 - mean SST = 305 K / dSST = 1.25 K if (sst_option == 15) then - mean_SST = 305 + mean_SST = 305 - TkFrz delta_SST = 1.25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) From 57fee69019b3ccc1ed5682966c486cf59bd430ad Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 3 Jul 2024 14:34:37 -0700 Subject: [PATCH 020/366] add TOA rad history variables --- .../eam/src/physics/crm/rrtmgp/radiation.F90 | 19 +++++++++++++++++++ .../eam/src/physics/rrtmgp/radiation.F90 | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/components/eam/src/physics/crm/rrtmgp/radiation.F90 b/components/eam/src/physics/crm/rrtmgp/radiation.F90 index b7b253d1b6c..b80ea0c2fe1 100644 --- a/components/eam/src/physics/crm/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/crm/rrtmgp/radiation.F90 @@ -768,6 +768,18 @@ subroutine radiation_init(state) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2494,6 +2506,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2531,6 +2544,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) diff --git a/components/eam/src/physics/rrtmgp/radiation.F90 b/components/eam/src/physics/rrtmgp/radiation.F90 index 0c715000951..5c87c3376d9 100644 --- a/components/eam/src/physics/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/rrtmgp/radiation.F90 @@ -767,6 +767,18 @@ subroutine radiation_init(state,pbuf) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2375,6 +2387,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2412,6 +2425,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) From d3fc03d65772ac2620dc19ba89446da1f7964dee Mon Sep 17 00:00:00 2001 From: mingxuanwupnnl Date: Sun, 7 Jul 2024 13:44:18 -0500 Subject: [PATCH 021/366] fix VBS bug in soa partition subroutine --- components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 b/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 index 6546677cac3..5f7f690cf62 100644 --- a/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 +++ b/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 @@ -3530,7 +3530,7 @@ subroutine mam_soaexch_vbs_1subarea( & ! convert sat vapor conc from ug/m^3 to mol/m^3 then to mol/liter tmpa = (c0_soa_298(ll)*1.0e-6_r8/mw_gas(igas)) * 1.0e-3_r8 ! calc sat vapor pressure (atm) from molar-conc and temp [ 0.082056 = gas constant in (atm/deg-K/(mol/liter)) ] - p0_soa_298(ll) = 0.082056_r8*tmpa*temp + p0_soa_298(ll) = 0.082056_r8*tmpa*298.0_r8 end do ! calc soa gas saturation molar-mixing-ratio at local temp and air-pressure From 7c182a67279b6499b34ba9d2d4aee6953b221c1e Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 27 Jun 2024 17:26:26 -0500 Subject: [PATCH 022/366] data land for moab driver use existing lnd_domain attribute in infodata we do not differentiate on coupler side if data land or not still need to send the proper land domain file --- .../data_comps/dlnd/src/dlnd_comp_mod.F90 | 181 +++++++++++++++++- .../data_comps/dlnd/src/lnd_comp_mct.F90 | 40 +++- 2 files changed, 216 insertions(+), 5 deletions(-) diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index bf723259be4..7768b2dfe81 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -29,6 +29,10 @@ module dlnd_comp_mod use dlnd_shr_mod , only: domain_fracname ! namelist input use dlnd_shr_mod , only: nullstr +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! id of moab lnd app + use iso_c_binding +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -100,6 +104,12 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & scmMode, scmlat, scmlon) ! !DESCRIPTION: initialize dlnd model +#ifdef HAVE_MOAB + use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & + iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo, iMOAB_WriteMesh +#endif implicit none ! !INPUT/OUTPUT PARAMETERS: @@ -135,6 +145,18 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & character(nec_len) :: nec_str ! elevation class, as character string character(*), parameter :: domain_fracname_unset = 'null' +#ifdef HAVE_MOAB + character*400 tagname + real(R8) latv, lonv + integer iv, tagindex, ilat, ilon !, arrsize, nfields ! ierr is already defined as integer above + real(R8), allocatable, target :: data(:) + integer(IN), pointer :: idata(:) ! temporary + real(r8), dimension(:), allocatable :: moab_vert_coords ! temporary +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif + !--- formats --- character(*), parameter :: F00 = "('(dlnd_comp_init) ',8a)" character(*), parameter :: F0L = "('(dlnd_comp_init) ',a, l2)" @@ -256,6 +278,119 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & call t_stopf('dlnd_initmctdom') +#ifdef HAVE_MOAB + ilat = mct_aVect_indexRA(ggrid%data,'lat') + ilon = mct_aVect_indexRA(ggrid%data,'lon') + allocate(moab_vert_coords(lsize*3)) + do iv = 1, lsize + lonv = ggrid%data%rAttr(ilon, iv) * SHR_CONST_PI/180. + latv = ggrid%data%rAttr(ilat, iv) * SHR_CONST_PI/180. + moab_vert_coords(3*iv-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*iv-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*iv )=SIN(latv) + enddo + + ! create the vertices with coordinates from MCT domain + ierr = iMOAB_CreateVertices(mlnid, lsize*3, 3, moab_vert_coords) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to create MOAB vertices in data lnd model') + + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, & + 0, & ! dense, integer + 1, & ! number of components + tagindex ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to retrieve GLOBAL_ID tag ') + + ! get list of global IDs for Dofs + call mct_gsMap_orderedPoints(gsMap, my_task, idata) + + ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsize, & + 0, & ! vertex type + idata) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( mlnid, lsize, idata ); + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to resolve shared entities') + + deallocate(moab_vert_coords) + deallocate(idata) + + ierr = iMOAB_UpdateMeshInfo( mlnid ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to update mesh info ') + + allocate(data(lsize)) + ierr = iMOAB_DefineTagStorage( mlnid, "area:aream:frac:mask"//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create tag: area:aream:frac:mask' ) + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'area'),:) + tagname='area'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to get area tag ') + + ! set the same data for aream (model area) as area + ! data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'aream'),:) + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set aream tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'mask'),:) + tagname='mask'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set mask tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'frac'),:) + tagname='frac'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set frac tag ') + + deallocate(data) + + ! define tags + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_x2l_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_x2l_fields tags ') + + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_l2x_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_l2x_fields tags ') +#ifdef MOABDEBUG + ! debug test + outfile = 'LndDataMesh.h5m'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! + ! write out the mesh file to disk + ierr = iMOAB_WriteMesh(mlnid, trim(outfile), trim(wopts)) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' ERROR in writing data mesh lnd ') + endif +#endif +#endif !---------------------------------------------------------------------------- ! Initialize MCT attribute vectors !---------------------------------------------------------------------------- @@ -339,8 +474,15 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & inst_suffix, logunit, case_name) ! !DESCRIPTION: run method for dlnd model - implicit none +#ifdef HAVE_MOAB +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif + use seq_flds_mod , only: seq_flds_l2x_fields + use seq_flds_mod , only: moab_set_tag_from_av +#endif + implicit none ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) , intent(in) :: EClock type(mct_aVect) , intent(inout) :: x2l @@ -366,6 +508,17 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & integer(IN) :: nu ! unit number logical :: write_restart ! restart now character(len=18) :: date_str +#ifdef HAVE_MOAB + real(R8), allocatable, target :: datam(:) + type(mct_list) :: temp_list + integer :: size_list, index_list, lsize + type(mct_string) :: mctOStr ! + character*400 tagname, mct_field +#ifdef MOABDEBUG + integer :: cur_dlnd_stepno, ierr + character*100 outfile, wopts, lnum +#endif +#endif character(*), parameter :: F00 = "('(dlnd_comp_run) ',8a)" character(*), parameter :: F04 = "('(dlnd_comp_run) ',2a,2i8,'s')" @@ -464,6 +617,32 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & call t_stopf('DLND_RUN') +#ifdef HAVE_MOAB + lsize = mct_avect_lsize(l2x) ! is it the same as mct_avect_lsize(avstrm) ? + allocate(datam(lsize)) ! + call mct_list_init(temp_list ,seq_flds_l2x_fields) + size_list=mct_list_nitem (temp_list) + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call moab_set_tag_from_av(tagname, l2x, index_list, mlnid, datam, lsize) ! loop over all a2x fields, not just a few + enddo + call mct_list_clean(temp_list) + deallocate(datam) ! maybe we should keep it around, deallocate at the final only? + +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_dlnd_stepno ) + write(lnum,"(I0.2)")cur_dlnd_stepno + outfile = 'dlnd_comp_run_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mlnid, outfile, wopts) + if (ierr > 0 ) then + write(logunit,*) 'Failed to write data lnd component state ' + endif +#endif +#endif + end subroutine dlnd_comp_run !=============================================================================== diff --git a/components/data_comps/dlnd/src/lnd_comp_mct.F90 b/components/data_comps/dlnd/src/lnd_comp_mct.F90 index f5193ca8458..8361a8522d8 100644 --- a/components/data_comps/dlnd/src/lnd_comp_mct.F90 +++ b/components/data_comps/dlnd/src/lnd_comp_mct.F90 @@ -16,7 +16,11 @@ module lnd_comp_mct use dlnd_comp_mod , only: dlnd_comp_init, dlnd_comp_run, dlnd_comp_final use dlnd_shr_mod , only: dlnd_shr_read_namelists use seq_flds_mod , only: seq_flds_x2l_fields, seq_flds_l2x_fields - +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! iMOAB app id for lnd + use iso_c_binding + use iMOAB , only: iMOAB_RegisterApplication +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -52,7 +56,9 @@ module lnd_comp_mct !=============================================================================== subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) - +#ifdef HAVE_MOAB + use shr_stream_mod, only: shr_stream_getDomainInfo, shr_stream_getFile +#endif ! !DESCRIPTION: initialize dlnd model implicit none @@ -78,6 +84,16 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) real(R8) :: scmLon = shr_const_SPVAL ! single column lon character(*), parameter :: subName = "(lnd_init_mct) " !------------------------------------------------------------------------------- +#ifdef HAVE_MOAB + character(CL) :: filePath ! generic file path + character(CL) :: fileName ! generic file name + character(CS) :: timeName ! domain file: time variable name + character(CS) :: lonName ! domain file: lon variable name + character(CS) :: latName ! domain file: lat variable name + character(CS) :: hgtName ! domain file: hgt variable name + character(CS) :: maskName ! domain file: mask variable name + character(CS) :: areaName ! domain file: area variable name +#endif ! Set cdata pointers call seq_cdata_setptrs(cdata, & @@ -146,13 +162,29 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) !---------------------------------------------------------------------------- ! Initialize dlnd !---------------------------------------------------------------------------- - +#ifdef HAVE_MOAB + ierr = iMOAB_RegisterApplication(trim("DLND")//C_NULL_CHAR, mpicom, compid, mlnid) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in registering data lnd comp' + call shr_sys_abort(subname//' ERROR in registering data lnd comp') + endif +#endif call dlnd_comp_init(Eclock, x2l, l2x, & seq_flds_x2l_fields, seq_flds_l2x_fields, & SDLND, gsmap, ggrid, mpicom, compid, my_task, master_task, & inst_suffix, inst_name, logunit, read_restart, & scmMode, scmlat, scmlon) - +#ifdef HAVE_MOAB + if (my_task == master_task) then + call shr_stream_getDomainInfo(SDLND%stream(1), filePath,fileName,timeName,lonName, & + latName,hgtName,maskName,areaName) + call shr_stream_getFile(filePath,fileName) + ! send path of data lnd domain to MOAB coupler. + call seq_infodata_PutData( infodata, lnd_domain=fileName) ! we use the same one for regular case + ! in regular case, it is copied from fatmlndfrc ; so we don't know if it is data land or not + write(logunit,*), ' filename: ', filename + endif +#endif !---------------------------------------------------------------------------- ! Fill infodata that needs to be returned from dlnd !---------------------------------------------------------------------------- From 2ad116ba0b787f1c9ab391eb3d95025a27cb19db Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 28 Jun 2024 09:12:11 -0500 Subject: [PATCH 023/366] domain file or land data case domain file is actually set, in shr_strdata_type%domainFile all the other data models use the first stream file for domain information land is the one that actually has that member set in the SDLND structure it is easier to get it from there, then from stream the domain file in regular land case is defined by fatmlndfrac file --- .../data_comps/dlnd/src/lnd_comp_mct.F90 | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/components/data_comps/dlnd/src/lnd_comp_mct.F90 b/components/data_comps/dlnd/src/lnd_comp_mct.F90 index 8361a8522d8..b699ec217f0 100644 --- a/components/data_comps/dlnd/src/lnd_comp_mct.F90 +++ b/components/data_comps/dlnd/src/lnd_comp_mct.F90 @@ -84,16 +84,6 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) real(R8) :: scmLon = shr_const_SPVAL ! single column lon character(*), parameter :: subName = "(lnd_init_mct) " !------------------------------------------------------------------------------- -#ifdef HAVE_MOAB - character(CL) :: filePath ! generic file path - character(CL) :: fileName ! generic file name - character(CS) :: timeName ! domain file: time variable name - character(CS) :: lonName ! domain file: lon variable name - character(CS) :: latName ! domain file: lat variable name - character(CS) :: hgtName ! domain file: hgt variable name - character(CS) :: maskName ! domain file: mask variable name - character(CS) :: areaName ! domain file: area variable name -#endif ! Set cdata pointers call seq_cdata_setptrs(cdata, & @@ -176,13 +166,9 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) scmMode, scmlat, scmlon) #ifdef HAVE_MOAB if (my_task == master_task) then - call shr_stream_getDomainInfo(SDLND%stream(1), filePath,fileName,timeName,lonName, & - latName,hgtName,maskName,areaName) - call shr_stream_getFile(filePath,fileName) - ! send path of data lnd domain to MOAB coupler. - call seq_infodata_PutData( infodata, lnd_domain=fileName) ! we use the same one for regular case + call seq_infodata_PutData( infodata, lnd_domain=SDLND%domainFile) ! we use the same one for regular case ! in regular case, it is copied from fatmlndfrc ; so we don't know if it is data land or not - write(logunit,*), ' filename: ', filename + write(logunit,*), ' use this land domain file: ', SDLND%domainFile endif #endif !---------------------------------------------------------------------------- From 9f0a28c35186b7f68daf7cf79b1643258ac73485 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 15 Jul 2024 10:33:50 -0500 Subject: [PATCH 024/366] apply weights is not used anymore in fractions iMOAB_ApplyScalarProjectionWeights is not used anymore all maps are done now with seq_map method, and there is only one instance of iMOAB_ApplyScalarProjectionWeights in E3SM using moab driver --- driver-moab/main/seq_frac_mct.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-moab/main/seq_frac_mct.F90 b/driver-moab/main/seq_frac_mct.F90 index a89fc0b9aa4..031845a102e 100644 --- a/driver-moab/main/seq_frac_mct.F90 +++ b/driver-moab/main/seq_frac_mct.F90 @@ -180,7 +180,7 @@ module seq_frac_mct use iMOAB, only : iMOAB_DefineTagStorage, iMOAB_SetDoubleTagStorage, & iMOAB_GetMeshInfo, iMOAB_SetDoubleTagStorageWithGid, iMOAB_WriteMesh, & - iMOAB_ApplyScalarProjectionWeights, iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & + iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & iMOAB_FreeSenderBuffers, iMOAB_GetDoubleTagStorage #ifdef MOABDEBUG use seq_comm_mct, only : num_moab_exports From 24d9107e254be2cb41421980d31ca34e368b30a4 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Wed, 7 Feb 2024 06:50:31 -0700 Subject: [PATCH 025/366] Add relativeVorticityAtSurface to highFrequencyStats --- components/mpas-ocean/cime_config/buildnml | 1 + .../src/analysis_members/Registry_high_frequency_output.xml | 5 ++++- .../src/analysis_members/mpas_ocn_high_frequency_output.F | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 4c7ed2c0a31..9ee046ae552 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -922,6 +922,7 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') diff --git a/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml b/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml index 5ab40af1923..67f0dcf60c4 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml @@ -27,6 +27,9 @@ + @@ -304,12 +307,12 @@ + - diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F index 60a5dececcb..60e7f47fa33 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F @@ -183,7 +183,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ integer, dimension(:,:), pointer :: edgesOnCell, cellsOnEdge, edgesOnEdge real (kind=RKIND) :: invAreaCell1, layerThicknessEdge1, coeff, weightedNormalVel, cellArea - real (kind=RKIND), dimension(:), pointer :: refBottomDepth, kineticEnergyAt250m, kineticEnergyAtSurface, relativeVorticityAt250m + real (kind=RKIND), dimension(:), pointer :: refBottomDepth, kineticEnergyAt250m, kineticEnergyAtSurface, relativeVorticityAt250m, relativeVorticityAtSurface real (kind=RKIND), dimension(:), pointer :: divergenceAt250m, relativeVorticityVertexAt250m real (kind=RKIND), dimension(:), pointer :: divergenceAtBottom,relativeVorticityAtBottom,kineticEnergyAtBottom real (kind=RKIND), dimension(:), pointer :: vertVelAt250m @@ -263,6 +263,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ call mpas_pool_get_array(highFrequencyOutputAMPool, 'kineticEnergyAt250m', kineticEnergyAt250m) call mpas_pool_get_array(highFrequencyOutputAMPool, 'kineticEnergyAtSurface', kineticEnergyAtSurface) call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAt250m', relativeVorticityAt250m) + call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAtSurface', relativeVorticityAtSurface) call mpas_pool_get_array(highFrequencyOutputAMPool, 'divergenceAt250m', divergenceAt250m) call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAtBottom', relativeVorticityAtBottom) call mpas_pool_get_array(highFrequencyOutputAMPool, 'divergenceAtBottom', divergenceAtBottom) @@ -418,6 +419,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ divergenceAt250m(:) = divergence(iLevel0250,:) relativeVorticityVertexAt250m(:) = relativeVorticity(iLevel0250,:) kineticEnergyAtSurface(:) = kineticEnergyCell(1,:) + relativeVorticityAtSurface(:) = relativeVorticityCell(1,:) activeTracersAtSurface(1,:) = activeTracers(1,1,:) activeTracersAtSurface(2,:) = activeTracers(2,1,:) activeTracersAt250m(1,:) = activeTracers(1,iLevel0250,:) From 6387dc4b78d92517018a9bf0e4c7bfa2762ed3b1 Mon Sep 17 00:00:00 2001 From: Hong-Yi Li Date: Mon, 26 Aug 2024 18:37:51 -0700 Subject: [PATCH 026/366] Fixing a bug causing model crash by avoiding negative channel storage --- .../src/riverroute/MOSART_physics_mod.F90 | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/components/mosart/src/riverroute/MOSART_physics_mod.F90 b/components/mosart/src/riverroute/MOSART_physics_mod.F90 index cad978577cd..132aff634cc 100644 --- a/components/mosart/src/riverroute/MOSART_physics_mod.F90 +++ b/components/mosart/src/riverroute/MOSART_physics_mod.F90 @@ -681,7 +681,9 @@ subroutine Euler ! subcycling within MOSART ends ! check for negative channel storage - if (negchan < -1.e-10) then + if (negchan < -1.e-10 .and. negchan >= -1.e-8) then + write(iulog,*) 'Warning: Small negative channel storage found! ',negchan + elseif(negchan < -1.e-8) then write(iulog,*) 'Error: Negative channel storage found! ',negchan call shr_sys_abort('mosart: negative channel storage') endif @@ -876,11 +878,7 @@ subroutine Routing_KW(iunit, nt, theDeltaT) TRunoff%erout(iunit,nt) = -TRunoff%vr(iunit,nt) * TRunoff%mr(iunit,nt) if(-TRunoff%erout(iunit,nt) > TINYVALUE .and. TRunoff%wr(iunit,nt) + & (TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%erout(iunit,nt)) * theDeltaT < TINYVALUE) then - if (sediflag) then - TRunoff%erout(iunit,nt) = -(TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%wr(iunit,nt)*MaxStorageDepleted/ theDeltaT) - else - TRunoff%erout(iunit,nt) = -(TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%wr(iunit,nt)/ theDeltaT) - end if + TRunoff%erout(iunit,nt) = -(TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%wr(iunit,nt)*MaxStorageDepleted/ theDeltaT) if(TRunoff%mr(iunit,nt) > 0._r8) then TRunoff%vr(iunit,nt) = -TRunoff%erout(iunit,nt) / TRunoff%mr(iunit,nt) end if @@ -918,16 +916,6 @@ subroutine Routing_KW(iunit, nt, theDeltaT) TRunoff%dwr(iunit,nt) = TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%erout(iunit,nt) + temp_gwl - !if(TRunoff%wr(iunit,nt) < TINYVALUE .and. abs(TRunoff%erout(iunit,nt))> TINYVALUE) then - ! write(unit=1111,fmt="(i10, 4(e20.11))") iunit, TRunoff%wr(iunit,nt), TRunoff%erout(iunit,nt), TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt), TRunoff%dwr(iunit,nt) - ! write(unit=1112,fmt="(2(i10), 4(e20.11))") iunit, TUnit%mask(iunit), TRunoff%vr(iunit,nt), TUnit%rlen(iunit), TUnit%rwidth(iunit), TUnit%areaTotal2(iunit)/TUnit%rwidth(iunit)/TUnit%rlen(iunit) - !end if - -! if(iunit==490 .and. nt==1) then -! write(unit=1111,fmt="(3(e20.11), 5(f12.4))") TUnit%areaTotal2(iunit),TUnit%areaTotal(iunit),TUnit%area(iunit),TUnit%rdepth(iunit), TUnit%rwidth(iunit), TUnit%rslp(iunit), TUnit%nr(iunit), TUnit%nt(iunit) -! write(unit=1112,fmt="(6(e20.11))") TRunoff%wr(iunit,nt), TRunoff%dwr(iunit,nt), TRunoff%erlateral(iunit,nt), TRunoff%erin(iunit,nt), TRunoff%erout(iunit,nt), temp_gwl -! end if - ! check for stability ! if(TRunoff%vr(iunit,nt) < -TINYVALUE .or. TRunoff%vr(iunit,nt) > 30) then ! write(iulog,*) "Numerical error inRouting_KW, ", iunit,nt,TRunoff%vr(iunit,nt) From ffb3b6103bc182b235a0c9392e56be41c801dc85 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 27 Aug 2024 15:10:10 -0500 Subject: [PATCH 027/366] initialize factors in area correction computations to 1.0 only intel on chrysalis complained about it It could be a cause for trouble later on --- driver-moab/main/component_mod.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/driver-moab/main/component_mod.F90 b/driver-moab/main/component_mod.F90 index 88d152a9568..3e8ba9042a5 100644 --- a/driver-moab/main/component_mod.F90 +++ b/driver-moab/main/component_mod.F90 @@ -748,6 +748,7 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe lsize = comp(1)%mblsize allocate(areas (lsize, 3)) ! lsize is along grid; read mask too allocate(factors (lsize, 2)) + factors = 1.0 ! initialize with 1.0 all factors; then maybe correct them ! get areas tagname='area:aream:mask'//C_NULL_CHAR arrsize = 3 * lsize From 6f601bfb6fe2948787de60d44de5a3fa2275bd28 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Tue, 3 Sep 2024 14:30:47 -0700 Subject: [PATCH 028/366] Adds notes about generating ELM land IC --- components/elm/docs/user-guide/index.md | 5 ++ components/elm/docs/user-guide/interpinic.md | 79 ++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 components/elm/docs/user-guide/interpinic.md diff --git a/components/elm/docs/user-guide/index.md b/components/elm/docs/user-guide/index.md index 7995e54acb3..d33278ce761 100644 --- a/components/elm/docs/user-guide/index.md +++ b/components/elm/docs/user-guide/index.md @@ -76,3 +76,8 @@ Using the above-mentioned settings: [FATES](fates.md) can be run in various modes with ELM through the use of namelist settings. The [FATES User's Guide section on namelist options](https://fates-users-guide.readthedocs.io/en/latest/user/Namelist-Options-and-Run-Time-Modes.html) provides guidance on enabling these different FATES run modes. + +## Generate land initial condition + +Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). + diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md new file mode 100644 index 00000000000..63f8c30b03d --- /dev/null +++ b/components/elm/docs/user-guide/interpinic.md @@ -0,0 +1,79 @@ +# Creating an ELM initial condition file + +An ELM initial condition (IC) file can be created by remapping an existing IC file from +one resolution to another using the `interpinic`, located at +`components/elm/tools/interpinic`. An ELM IC file is in the same format as an ELM restart file. +The composet of the remapped IC file will be the same as that of the input IC file. +So, for a new ELM SP-mode IC file, use an ELM input file corresponding to the SP-mode. + +The steps involved in creating a new IC files are as follows: + +1. Identifying an input ELM IC or restart file that will be remapped. +2. Obtaining an ELM restart file at the new resolution. +3. Compiling `interpinic` on the machine of interest. +4. Running `interpinic` to perform the interpolation. + +The notes below provide an example of creating 1850 ELM IC file for the NARRM grid using E3SM v3 LR piControl from year = 0101. These notes are provided for Chrysalis. + +### 1. Identification of the input ELM IC file + +The identified input land condition file for this case is the following: + +``` +/lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc +``` + +## 2. Obtaining an ELM restart file + +Using an existing NARRM land IC and making a copy of it + +``` +cd components/elm/tools/interpinic + +cp /lcrc/group/e3sm/data/inputdata/lnd/clm2/initdata_map/elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1870-01-01-00000.c20240704.nc \ +elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c`date "+%Y%m%d"`.nc +``` + +## 3. Compiling `interpinic` + +``` +# Load relevant modules +cd +eval $(./cime/CIME/Tools/get_case_env/get_case_env) + +# change directory +cd components/elm/tools/interpinic/src + +export USER_LDFLAGS="-L$NETCDF_C_DIR/lib -lnetcdf -L$NETCDF_F_DIR/lib -lnetcdff -L$HDF5_DIR/lib -lhdf5" + +USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 + +``` + +## 4. Run `interpinic` + +The `interpinic ` can then be run via the following batch job (e.g., `remap.r025_RRSwISC6to18E3r4.1850.batch`) to generate the initial condition. + +``` +>cat remap.r025_RRSwISC6to18E3r4.1850.batch + +#!/bin/sh +#SBATCH --job-name=remap +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --time 24:00:00 +#SBATCH -p slurm +#SBATCH --account esmd + +# Load relevant modules. +cd +eval $(./cime/CIME/Tools/get_case_env/get_case_env) + +# Change dir to `interpinic` +cd components/elm/tools/interpinic/src + +srun -n 1 ./interpinic \ +-i /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc \ +-o elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c20240903.nc +``` + From c99179079aee1d1b793ecf07530252046e2e40d6 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:19:19 -0600 Subject: [PATCH 029/366] HOMME: add aurora, polaris, spot machine files --- .../homme/cmake/machineFiles/aurora-aot.cmake | 64 ++++++++++++++++ .../homme/cmake/machineFiles/aurora-jit.cmake | 58 +++++++++++++++ .../cmake/machineFiles/chrysalis-bfb.cmake | 2 + .../homme/cmake/machineFiles/chrysalis.cmake | 2 + .../homme/cmake/machineFiles/polaris-a100.sh | 74 +++++++++++++++++++ .../cmake/machineFiles/spot-aot-AB2.cmake | 63 ++++++++++++++++ 6 files changed, 263 insertions(+) create mode 100644 components/homme/cmake/machineFiles/aurora-aot.cmake create mode 100644 components/homme/cmake/machineFiles/aurora-jit.cmake create mode 100644 components/homme/cmake/machineFiles/polaris-a100.sh create mode 100644 components/homme/cmake/machineFiles/spot-aot-AB2.cmake diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake new file mode 100644 index 00000000000..b6fe34a78d7 --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -0,0 +1,64 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + + +SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/mar05-aot/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +# -fsycl-link-huge-device-code for theta to get build +#JIT flags +#SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +#SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") + +#AOT flags +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fsycl-link-huge-device-code -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + diff --git a/components/homme/cmake/machineFiles/aurora-jit.cmake b/components/homme/cmake/machineFiles/aurora-jit.cmake new file mode 100644 index 00000000000..1941fa9eb3f --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-jit.cmake @@ -0,0 +1,58 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + + + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/jan03-2024/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +# -fsycl-link-huge-device-code for theta to get build +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") + +SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + diff --git a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake index b9f0d41a060..fa1f1ac545c 100644 --- a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake +++ b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake @@ -17,6 +17,8 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") +SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") + # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/chrysalis.cmake b/components/homme/cmake/machineFiles/chrysalis.cmake index 68ff76ec808..97bc682c954 100644 --- a/components/homme/cmake/machineFiles/chrysalis.cmake +++ b/components/homme/cmake/machineFiles/chrysalis.cmake @@ -17,6 +17,8 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") +SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") + # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/polaris-a100.sh b/components/homme/cmake/machineFiles/polaris-a100.sh new file mode 100644 index 00000000000..2b63c61a55e --- /dev/null +++ b/components/homme/cmake/machineFiles/polaris-a100.sh @@ -0,0 +1,74 @@ +#Currently Loaded Modules: +# 1) craype-x86-rome 6) craype/2.7.15 11) cray-libpals/1.1.7 16) nvhpc-mixed/21.9 +# 2) libfabric/1.11.0.4.125 7) cray-dsmml/0.2.2 12) PrgEnv-gnu/8.3.3 17) cudatoolkit-standalone/11.6.2 +# 3) craype-network-ofi 8) cray-pmi/6.1.2 13) gnu-parallel/2021-09-22 18) cmake/3.23.2 +# 4) perftools-base/22.05.0 9) cray-pmi-lib/6.0.17 14) gcc/11.2.0 +# 5) craype-accel-nvidia80 10) cray-pals/1.1.7 15) cray-mpich/8.1.16 + + + +#SET(HOMMEXX_EXEC_SPACE CUDA CACHE STRING "") +#SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") +#SET(HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING "") + +# cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 +#SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET(PNETCDF_DIR $ENV{CRAY_PARALLEL_NETCDF_DIR} CACHE FILEPATH "") +#SET(HDF5_DIR $ENV{CRAY_HDF5_PARALLEL_PREFIX} CACHE FILEPATH "") + +#for scorpio +#SET (NetCDF_C_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET (NetCDF_Fortran_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK FALSE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +SET(CUDA_BUILD TRUE CACHE BOOL "") + +#SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA ON CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "") +SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") +#SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") # works, and perf same if both AMPERE80 and ZEN2 are on +#SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") +SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") +#SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") +SET(CMAKE_C_COMPILER "cc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "${CMAKE_CURRENT_SOURCE_DIR}/../../externals/kokkos/bin/nvcc_wrapper" CACHE STRING "") + +# Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER + +SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") + +SET(ENABLE_OPENMP OFF CACHE BOOL "") +SET(ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +SET(ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +SET(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "") + +#SET(HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +SET(USE_NUM_PROCS 4 CACHE STRING "") + +SET(USE_MPIEXEC "srun" CACHE STRING "") +#SET(CPRNC_DIR /global/cfs/cdirs/e3sm/tools/cprnc CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/spot-aot-AB2.cmake b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake new file mode 100644 index 00000000000..23fad2361cc --- /dev/null +++ b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake @@ -0,0 +1,63 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + +SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") + +SET (HOMMEXX_MPI_ON_DEVICE TRUE CACHE BOOL "") + +#SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/june22-2024-aot/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET (NetCDF_Fortran_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +SET (NetCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +#SET(CMAKE_CXX_STANDARD 17) +SET(CMAKE_CXX_STANDARD 17 CACHE STRING "CXX Standard") + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +#-fpscomp does not actually solve the issue with bools in here,another suggestion was -fp-model=precise, not working either +SET(ADD_Fortran_FLAGS " -fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS " -std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + From 1379db52a29ff31764b6268eae1f88fbedbae252 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:29:27 -0600 Subject: [PATCH 030/366] HOMME: CMake and CPP mods to support SYCL backend --- components/homme/CMakeLists.txt | 8 +++++--- components/homme/cmake/HommeMacros.cmake | 8 +++++++- components/homme/src/share/cxx/Config.hpp | 2 +- components/homme/src/share/cxx/ExecSpaceDefs.hpp | 4 ++++ .../homme/test_execs/share_kokkos_ut/CMakeLists.txt | 4 ++-- .../homme/test_execs/thetal_kokkos_ut/CMakeLists.txt | 3 +++ 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 80a89a29691..6fe81180ab5 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -206,7 +206,9 @@ IF (HOMME_USE_KOKKOS) STRING (TOUPPER ${HOMMEXX_EXEC_SPACE} HOMMEXX_EXEC_SPACE_UPPER) - IF (HOMMEXX_EXEC_SPACE_UPPER STREQUAL "HIP") + IF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "SYCL") + SET (HOMMEXX_SYCL_SPACE ON) + ELSEIF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "HIP") SET (HOMMEXX_HIP_SPACE ON) ELSEIF (HOMMEXX_EXEC_SPACE_UPPER STREQUAL "CUDA") SET (HOMMEXX_CUDA_SPACE ON) @@ -303,7 +305,7 @@ SET (HOMMEXX_ENABLE_GPU_F90 FALSE) IF (HOMME_USE_KOKKOS) - IF (CUDA_BUILD OR HIP_BUILD) + IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET (DEFAULT_VECTOR_SIZE 1) SET (HOMMEXX_ENABLE_GPU TRUE) SET (HOMMEXX_ENABLE_GPU_F90 TRUE) @@ -312,7 +314,7 @@ IF (HOMME_USE_KOKKOS) ENDIF() SET (HOMMEXX_VECTOR_SIZE ${DEFAULT_VECTOR_SIZE} CACHE STRING - "If AVX or Cuda or HIP don't take priority, use this software vector size.") + "If AVX or Cuda or HIP or SYCL don't take priority, use this software vector size.") IF (CMAKE_BUILD_TYPE_UPPER MATCHES "DEBUG" OR CMAKE_BUILD_TYPE_UPPER MATCHES "RELWITHDEBINFO") SET (HOMMEXX_DEBUG ON) diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 6d073dbbe83..5610947cb29 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -112,7 +112,13 @@ macro(createTestExec execName execType macroNP macroNC ADD_DEFINITIONS(-DHAVE_CONFIG_H) ADD_EXECUTABLE(${execName} ${EXEC_SOURCES}) - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + + if(SUNSPOT_MACHINE) + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX) + else() + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + endif() + IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() diff --git a/components/homme/src/share/cxx/Config.hpp b/components/homme/src/share/cxx/Config.hpp index 684f9143bea..b204b1dbd04 100644 --- a/components/homme/src/share/cxx/Config.hpp +++ b/components/homme/src/share/cxx/Config.hpp @@ -21,7 +21,7 @@ # endif #endif -#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE && ! defined HOMMEXX_HIP_SPACE +#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE && ! defined HOMMEXX_HIP_SPACE && ! defined HOMMEXX_SYCL_SPACE # define HOMMEXX_DEFAULT_SPACE #endif diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.hpp b/components/homme/src/share/cxx/ExecSpaceDefs.hpp index cd6649c7ab2..82f5e803801 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.hpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.hpp @@ -34,6 +34,10 @@ using HommexxGPU = Kokkos::Cuda; using HommexxGPU = Kokkos::Experimental::HIP; #endif +#ifdef KOKKOS_ENABLE_SYCL +using HommexxGPU = Kokkos::Experimental::SYCL; +#endif + #else using HommexxGPU = void; #endif diff --git a/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt index 3fbeff9f6f2..bc788462ce6 100644 --- a/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt @@ -10,7 +10,7 @@ SET(UTILS_TIMING_DIRS ${UTILS_TIMING_SRC_DIR} ${UTILS_TIMING_BIN_DIR}) # Note: need CUDA_BUILD and HOMMEXX_BFB_TESTING here, since the share # unit tests do not include a config.h file SET (COMMON_DEFINITIONS NP=4 NC=4) -IF (CUDA_BUILD OR HIP_BUILD) +IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET(COMMON_DEFINITIONS ${COMMON_DEFINITIONS} HOMMEXX_ENABLE_GPU_F90) ENDIF() IF (HOMMEXX_BFB_TESTING) @@ -158,7 +158,7 @@ ELSE() SET (NUM_CPUS 1) ENDIF() cxx_unit_test (sphere_op_ut "${SPHERE_OP_UT_F90_SRCS}" "${SPHERE_OP_UT_CXX_SRCS}" "${SPHERE_OP_UT_INCLUDE_DIRS}" "${CONFIG_DEFINES}" ${NUM_CPUS}) -endif () +endif () #BFB ### Limiters unit test ### diff --git a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt index 205635e918c..e8bf5e20bd0 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt @@ -11,6 +11,8 @@ SET(UTILS_TIMING_BIN_DIR ${HOMME_BINARY_DIR}/utils/cime/CIME/non_py/src/timing) THETAL_KOKKOS_SETUP() # This is needed to compile the lib and test executables with the correct options +#these vars shared between all targets, so changing one var +#for one test only won't work, config is built once and for the last test SET(THIS_CONFIG_IN ${HOMME_SOURCE_DIR}/src/theta-l_kokkos/config.h.cmake.in) SET(THIS_CONFIG_HC ${CMAKE_CURRENT_BINARY_DIR}/config.h.c) SET(THIS_CONFIG_H ${CMAKE_CURRENT_BINARY_DIR}/config.h) @@ -18,6 +20,7 @@ SET (NUM_POINTS 4) SET (NUM_PLEV 12) SET (QSIZE_D 4) SET (PIO_INTERP TRUE) + HommeConfigFile (${THIS_CONFIG_IN} ${THIS_CONFIG_HC} ${THIS_CONFIG_H} ) ADD_LIBRARY(thetal_kokkos_ut_lib From c7320a6c5e31c161cf8c48ecb72d0ad792506ddf Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:30:22 -0600 Subject: [PATCH 031/366] HOMME: replace some printf with Kokkos::printf --- components/homme/src/share/cxx/utilities/BfbUtils.hpp | 2 +- .../homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp | 4 ++-- .../homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/homme/src/share/cxx/utilities/BfbUtils.hpp b/components/homme/src/share/cxx/utilities/BfbUtils.hpp index e3570874e26..7fb4d042f7f 100644 --- a/components/homme/src/share/cxx/utilities/BfbUtils.hpp +++ b/components/homme/src/share/cxx/utilities/BfbUtils.hpp @@ -64,7 +64,7 @@ KOKKOS_INLINE_FUNCTION ScalarType int_pow (ScalarType val, int k) { constexpr int max_shift = 30; if (k<0) { - printf ("k = %d\n",k); + Kokkos::printf ("k = %d\n",k); Kokkos::abort("int_pow implemented only for k>=0.\n"); } diff --git a/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp index ace1ba92014..d1676907972 100644 --- a/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp @@ -382,8 +382,8 @@ struct DirkFunctorImpl { kv.team_barrier(); if (it >= maxiter) { - printf("[DIRK] WARNING! Newton reached max iteration count," - " with deltaerr = %3.17f\n", deltaerr); + Kokkos::printf("[DIRK] WARNING! Newton reached max iteration count," + " with deltaerr = %3.17f\n", deltaerr); nerr = 1; } diff --git a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp index cd3bf7c3252..7914c0a60e3 100644 --- a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp @@ -141,8 +141,8 @@ struct LimiterFunctor { [&](const int k,Real& result) { #ifndef HOMMEXX_BFB_TESTING if(diff_as_real(k) < 0){ - printf("WARNING:CAAR: dp3d too small. k=%d, dp3d(k)=%f, dp0=%f \n", - k+1,dp_as_real(k),dp0_as_real(k)); + Kokkos::printf("WARNING:CAAR: dp3d too small. k=%d, dp3d(k)=%f, dp0=%f \n", + k+1,dp_as_real(k),dp0_as_real(k)); } #endif result = result<=diff_as_real(k) ? result : diff_as_real(k); @@ -202,8 +202,8 @@ struct LimiterFunctor { for (int ivec=0; ivec Date: Thu, 5 Sep 2024 16:33:09 -0600 Subject: [PATCH 032/366] HOMME: prefer to use int to bool in a few places --- components/homme/src/share/control_mod.F90 | 1 + components/homme/src/share/cxx/GllFvRemap.cpp | 4 +- components/homme/src/share/cxx/GllFvRemap.hpp | 4 +- .../homme/src/share/cxx/GllFvRemapImpl.cpp | 8 ++-- .../homme/src/share/cxx/GllFvRemapImpl.hpp | 5 ++- .../homme/src/share/cxx/SimulationParams.hpp | 4 +- components/homme/src/share/gllfvremap_mod.F90 | 14 +++---- components/homme/src/share/namelist_mod.F90 | 6 +++ .../src/theta-l_kokkos/cxx/CamForcing.cpp | 2 +- .../theta-l_kokkos/cxx/EquationOfState.hpp | 4 +- .../src/theta-l_kokkos/cxx/ForcingFunctor.hpp | 4 +- .../cxx/HyperviscosityFunctorImpl.cpp | 8 +++- .../cxx/HyperviscosityFunctorImpl.hpp | 2 +- .../cxx/cxx_f90_interface_theta.cpp | 19 +++++----- .../src/theta-l_kokkos/prim_driver_mod.F90 | 38 ++++++++++++------- .../src/theta-l_kokkos/theta_f2c_mod.F90 | 12 +++--- 16 files changed, 79 insertions(+), 56 deletions(-) diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 0e9494f5a6c..9c3c599b232 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -43,6 +43,7 @@ module control_mod ! flag used by preqx, theta-l and theta-c models ! should be renamed to "hydrostatic_mode" logical, public :: theta_hydrostatic_mode + integer, public :: theta_hydrostatic_mode_integer integer, public :: tstep_type= 5 ! preqx timestepping options diff --git a/components/homme/src/share/cxx/GllFvRemap.cpp b/components/homme/src/share/cxx/GllFvRemap.cpp index e36dbc14d74..a8f564958d4 100644 --- a/components/homme/src/share/cxx/GllFvRemap.cpp +++ b/components/homme/src/share/cxx/GllFvRemap.cpp @@ -16,7 +16,7 @@ namespace Homme { void init_gllfvremap_c (int nelemd, int np, int nf, int nf_max, - bool theta_hydrostatic_mode, + int theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f) { auto& c = Context::singleton(); @@ -52,7 +52,7 @@ void GllFvRemap::init_boundary_exchanges () { } void GllFvRemap -::init_data (const int nf, const int nf_max, bool theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f) { m_impl->init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, diff --git a/components/homme/src/share/cxx/GllFvRemap.hpp b/components/homme/src/share/cxx/GllFvRemap.hpp index 07e4bf58a90..2adff0aeaa9 100644 --- a/components/homme/src/share/cxx/GllFvRemap.hpp +++ b/components/homme/src/share/cxx/GllFvRemap.hpp @@ -40,7 +40,7 @@ class GllFvRemap { typedef Phys2T::const_type CPhys2T; typedef Phys3T::const_type CPhys3T; - void init_data(const int nf, const int nf_max, bool theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f); @@ -81,7 +81,7 @@ class GllFvRemap { extern "C" void init_gllfvremap_c(int nelemd, int np, int nf, int nf_max, - const bool theta_hydrostatic_mode, + const int theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index 6148f69cfa9..d4ab5c89f51 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -131,7 +131,7 @@ void GllFvRemapImpl::init_boundary_exchanges () { template using FV = Kokkos::View; void GllFvRemapImpl -::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r) { using Kokkos::create_mirror_view; @@ -142,7 +142,7 @@ ::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, " nf must be > 1.", Errors::err_not_implemented); auto& sp = Context::singleton().get(); - m_data.use_moisture = sp.moisture == MoistDry::MOIST; + m_data.use_moisture = sp.use_moisture; // Only in the unit test gllfvremap_ut does theta_hydrostatic_mode not already // == sp.theta_hydrostatic_mode. m_data.theta_hydrostatic_mode = sp.theta_hydrostatic_mode = theta_hydrostatic_mode; @@ -395,7 +395,7 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c const auto hvcoord = m_hvcoord; const bool use_moisture = m_data.use_moisture; - const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; const bool want_dp_fv_out = dp_fv_out_ptr != nullptr; VPhys2T dp_fv_out; @@ -605,7 +605,7 @@ run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, const auto fT = m_forcing.m_ft; const auto hvcoord = m_hvcoord; const auto dp3d = m_state.m_dp3d; - const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; EquationOfState eos; eos.init(theta_hydrostatic_mode, hvcoord); ElementOps ops; ops.init(hvcoord); const auto tu_ne = m_tu_ne; diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index 11738b2bf45..7388fddb123 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -60,7 +60,8 @@ struct GllFvRemapImpl { struct Data { int nelemd, qsize, nf2, n_dss_fld; - bool use_moisture, theta_hydrostatic_mode; + bool use_moisture; + int theta_hydrostatic_mode; static constexpr int nbuf1 = 2, nbuf2 = 1; Buf1 buf1[nbuf1]; @@ -107,7 +108,7 @@ struct GllFvRemapImpl { void init_buffers(const FunctorsBuffersManager& fbm); void init_boundary_exchanges(); - void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r); diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index b435911da2e..4f36962b16c 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -23,7 +23,7 @@ struct SimulationParams void print(std::ostream& out = std::cout); TimeStepType time_step_type; - MoistDry moisture; + bool use_moisture; RemapAlg remap_alg; TestCase test_case; ForcingAlg ftype = ForcingAlg::FORCING_OFF; @@ -77,7 +77,7 @@ inline void SimulationParams::print (std::ostream& out) { out << "\n************** CXX SimulationParams **********************\n\n"; out << " time_step_type: " << etoi(time_step_type) << "\n"; - out << " moisture: " << (moisture==MoistDry::DRY ? "dry" : "moist") << "\n"; + out << " use_moisture: " << (use_moisture ? "moist" : "dry") << "\n"; out << " remap_alg: " << etoi(remap_alg) << "\n"; out << " test case: " << etoi(test_case) << "\n"; out << " ftype: " << etoi(ftype) << "\n"; diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e0e0fa6c4da..e927f04aba0 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -265,22 +265,22 @@ end subroutine gfr_init subroutine gfr_init_hxx() bind(c) #if KOKKOS_TARGET - use control_mod, only: theta_hydrostatic_mode - use iso_c_binding, only: c_bool + use control_mod, only: theta_hydrostatic_mode_integer + use iso_c_binding, only: c_int interface - subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode, & + subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode_integer, & fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f) bind(c) - use iso_c_binding, only: c_bool, c_int, c_double + use iso_c_binding, only: c_int, c_double integer (c_int), value, intent(in) :: nelemd, np, nf, nf_max - logical (c_bool), value, intent(in) :: theta_hydrostatic_mode + integer (c_int), value, intent(in) :: theta_hydrostatic_mode_integer real (c_double), dimension(nf*nf,nelemd), intent(in) :: fv_metdet real (c_double), dimension(np,np,nf_max*nf_max), intent(in) :: g2f_remapd real (c_double), dimension(nf_max*nf_max,np,np), intent(in) :: f2g_remapd real (c_double), dimension(nf*nf,2,2,nelemd), intent(in) :: D_f, Dinv_f end subroutine init_gllfvremap_c end interface - logical (c_bool) :: thm - thm = theta_hydrostatic_mode + integer (c_int) :: thm + thm = theta_hydrostatic_mode_integer call init_gllfvremap_c(nelemd, np, gfr%nphys, nphys_max, thm, & gfr%fv_metdet, gfr%g2f_remapd, gfr%f2g_remapd, gfr%D_f, gfr%Dinv_f) #endif diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 1d47090182b..a3edaa07e23 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -41,6 +41,7 @@ module namelist_mod runtype, & integration, & ! integration method theta_hydrostatic_mode, & + theta_hydrostatic_mode_integer, & transport_alg , & ! SE Eulerian, classical SL, cell-integrated SL semi_lagrange_cdr_alg, & ! see control_mod for semi_lagrange_* descriptions semi_lagrange_cdr_check, & @@ -452,8 +453,10 @@ subroutine readnl(par) planar_slice = .false. theta_hydrostatic_mode = .true. ! for preqx, this must be .true. + theta_hydrostatic_mode_integer = 1 ! for preqx, this must be .true. #if ( defined MODEL_THETA_C || defined MODEL_THETA_L ) theta_hydrostatic_mode = .false. ! default NH + theta_hydrostatic_mode_integer = 0 ! default NH #endif @@ -850,7 +853,10 @@ subroutine readnl(par) call MPI_bcast(case_planar_bubble,1,MPIlogical_t,par%root,par%comm,ierr) #endif +if(theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 1 +if(.not. theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 0 call MPI_bcast(theta_hydrostatic_mode ,1,MPIlogical_t,par%root,par%comm,ierr) + call MPI_bcast(theta_hydrostatic_mode_integer ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(transport_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_check ,1,MPIlogical_t,par%root,par%comm,ierr) diff --git a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp index 02b999db16e..bd7cee3e7c0 100644 --- a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp @@ -33,7 +33,7 @@ static void apply_cam_forcing_tracers(const Real dt, ForcingFunctor& ff, if ( p.ftype == ForcingAlg::FORCING_2) adjustment = true; #endif - ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.moisture); + ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.use_moisture); GPTLstop("ApplyCAMForcing_tracers"); } diff --git a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp index dd97720f1be..a50a28d58f5 100644 --- a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp @@ -23,7 +23,7 @@ class EquationOfState { EquationOfState () = default; - void init (const bool theta_hydrostatic_mode, + void init (const int theta_hydrostatic_mode, const HybridVCoord& hvcoord) { m_theta_hydrostatic_mode = theta_hydrostatic_mode; m_hvcoord = hvcoord; @@ -250,7 +250,7 @@ class EquationOfState { public: - bool m_theta_hydrostatic_mode; + int m_theta_hydrostatic_mode; HybridVCoord m_hvcoord; }; diff --git a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp index 28a702c1d27..00fa1deef66 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp @@ -236,7 +236,7 @@ class ForcingFunctor }); } - void tracers_forcing (const Real dt, const int np1, const int np1_qdp, const bool adjustment, const MoistDry moisture) { + void tracers_forcing (const Real dt, const int np1, const int np1_qdp, const bool adjustment, const bool use_moisture) { // The Functor needs to be fully setup to use this function assert (is_setup); @@ -245,7 +245,7 @@ class ForcingFunctor m_np1_qdp = np1_qdp; m_adjustment = adjustment; - m_moist = (moisture==MoistDry::MOIST); + m_moist = use_moisture; Kokkos::parallel_for("temperature, NH perturb press, FQps",m_policy_tracers_pre,*this); Kokkos::fence(); diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index 046e6f9956d..55792051d33 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -118,9 +118,13 @@ void HyperviscosityFunctorImpl::init_params(const SimulationParams& params) m_eos.init(params.theta_hydrostatic_mode,m_hvcoord); #ifdef HOMMEXX_BFB_TESTING - m_process_nh_vars = true; + m_process_nh_vars = 1; #else - m_process_nh_vars = !params.theta_hydrostatic_mode; + if (params.theta_hydrostatic_mode){ + m_process_nh_vars = 0; + }else{ + m_process_nh_vars = 1; + } #endif } diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp index a55ecbb365f..993d525422f 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp @@ -397,7 +397,7 @@ class HyperviscosityFunctorImpl Buffers m_buffers; HybridVCoord m_hvcoord; - bool m_process_nh_vars; + int m_process_nh_vars; // Policies Kokkos::TeamPolicy m_policy_update_states; diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index ec4e2cbe632..40c4ae64dc9 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -43,12 +43,13 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const Real& nu, const Real& nu_p, const Real& nu_q, const Real& nu_s, const Real& nu_div, const Real& nu_top, const int& hypervis_order, const int& hypervis_subcycle, const int& hypervis_subcycle_tom, const double& hypervis_scaling, const double& dcmip16_mu, - const int& ftype, const int& theta_adv_form, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, - const bool& use_cpstar, const int& transport_alg, const bool& theta_hydrostatic_mode, const char** test_case, + const int& ftype, const int& theta_adv_form, const int& prescribed_wind, const int& use_moisture, const int& disable_diagnostics, + const int& use_cpstar, const int& transport_alg, const int& theta_hydrostatic_mode, const char** test_case, const int& dt_remap_factor, const int& dt_tracer_factor, - const double& scale_factor, const double& laplacian_rigid_factor, const int& nsplit, const bool& pgrad_correction, + const double& scale_factor, const double& laplacian_rigid_factor, const int& nsplit, const int& pgrad_correction, const double& dp3d_thresh, const double& vtheta_thresh, const int& internal_diagnostics_level) { + // Check that the simulation options are supported. This helps us in the future, since we // are currently 'assuming' some option have/not have certain values. As we support for more // options in the C++ build, we will remove some checks @@ -111,16 +112,16 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_subcycle = hypervis_subcycle; params.hypervis_subcycle_tom = hypervis_subcycle_tom; params.hypervis_scaling = hypervis_scaling; - params.disable_diagnostics = disable_diagnostics; - params.moisture = (moisture ? MoistDry::MOIST : MoistDry::DRY); - params.use_cpstar = use_cpstar; + params.disable_diagnostics = (bool)disable_diagnostics; + params.use_moisture = (bool)use_moisture; + params.use_cpstar = (bool)use_cpstar; params.transport_alg = transport_alg; - params.theta_hydrostatic_mode = theta_hydrostatic_mode; + params.theta_hydrostatic_mode = (bool)theta_hydrostatic_mode; params.dcmip16_mu = dcmip16_mu; params.nsplit = nsplit; params.scale_factor = scale_factor; params.laplacian_rigid_factor = laplacian_rigid_factor; - params.pgrad_correction = pgrad_correction; + params.pgrad_correction = (bool)pgrad_correction; params.dp3d_thresh = dp3d_thresh; params.vtheta_thresh = vtheta_thresh; params.internal_diagnostics_level = internal_diagnostics_level; @@ -304,7 +305,7 @@ void init_elements_c (const int& num_elems) c.create_ref(e.m_forcing); } -void init_functors_c (const bool& allocate_buffer) +void init_functors_c (const int& allocate_buffer) { auto& c = Context::singleton(); diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 96b42314453..262ba19f4b7 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -64,7 +64,7 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) end subroutine prim_init2 subroutine prim_create_c_data_structures (tl, hvcoord, mp) - use iso_c_binding, only : c_loc, c_ptr, c_bool, C_NULL_CHAR + use iso_c_binding, only : c_loc, c_ptr, C_NULL_CHAR use theta_f2c_mod, only : init_reference_element_c, init_simulation_params_c, & init_time_level_c, init_hvcoord_c, init_elements_c use time_mod, only : TimeLevel_t, nsplit @@ -73,7 +73,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) nu, nu_p, nu_q, nu_s, nu_div, nu_top, vert_remap_q_alg, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom,& hypervis_scaling, & - ftype, prescribed_wind, moisture, disable_diagnostics, & + ftype, prescribed_wind, use_moisture, disable_diagnostics, & use_cpstar, transport_alg, theta_hydrostatic_mode, & dcmip16_mu, theta_advect_form, test_case, & MAX_STRING_LEN, dt_remap_factor, dt_tracer_factor, & @@ -93,6 +93,8 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) type (c_ptr) :: hybrid_am_ptr, hybrid_ai_ptr, hybrid_bm_ptr, hybrid_bi_ptr character(len=MAX_STRING_LEN), target :: test_name + integer :: disable_diagnostics_int, theta_hydrostatic_mode_int, use_moisture_int + ! Initialize the C++ reference element structure (i.e., pseudo-spectral deriv matrix and ref element mass matrix) dvv = deriv1%dvv elem_mp = mp @@ -100,22 +102,30 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) ! Fill the simulation params structures in C++ test_name = TRIM(test_case) // C_NULL_CHAR + + if (disable_diagnostics) disable_diagnostics_int=1 + if (.not.disable_diagnostics) disable_diagnostics_int=0 + if (use_moisture) use_moisture_int=1 + if (.not.use_moisture) use_moisture_int=0 + if(theta_hydrostatic_mode) theta_hydrostatic_mode_int=1 + if(.not.theta_hydrostatic_mode) theta_hydrostatic_mode_int=0 + call init_simulation_params_c (vert_remap_q_alg, limiter_option, rsplit, qsplit, tstep_type, & qsize, statefreq, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & hypervis_scaling, & dcmip16_mu, ftype, theta_advect_form, & - LOGICAL(prescribed_wind==1,c_bool), & - LOGICAL(moisture/="dry",c_bool), & - LOGICAL(disable_diagnostics,c_bool), & - LOGICAL(use_cpstar==1,c_bool), & + prescribed_wind, & + use_moisture_int, & + disable_diagnostics_int, & + use_cpstar, & transport_alg, & - LOGICAL(theta_hydrostatic_mode,c_bool), & + theta_hydrostatic_mode_int, & c_loc(test_name), & dt_remap_factor, dt_tracer_factor, & scale_factor, laplacian_rigid_factor, & nsplit, & - LOGICAL(pgrad_correction==1,c_bool), & + pgrad_correction, & dp3d_thresh, vtheta_thresh, internal_diagnostics_level) ! Initialize time level structure in C++ @@ -343,21 +353,21 @@ subroutine prim_init_elements_views (elem) end subroutine prim_init_elements_views subroutine prim_init_kokkos_functors (allocate_buffer) - use iso_c_binding, only : c_bool + use iso_c_binding, only : c_int use theta_f2c_mod, only : init_functors_c, init_boundary_exchanges_c - ! ! Optional Input ! - logical(kind=c_bool), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally - + integer, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + integer(kind=c_int) :: dummy ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, ! let Homme internally allocate buffers if (present(allocate_buffer)) then - call init_functors_c (logical(allocate_buffer,c_bool)) + call init_functors_c (allocate_buffer) else - call init_functors_c (logical(.true.,c_bool)) + dummy=1; + call init_functors_c (dummy) endif ! Initialize boundary exchange structure in C++ diff --git a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 index 7a4c0424807..ba39bb03c22 100644 --- a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 +++ b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 @@ -11,14 +11,14 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, qsize, state_frequency, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & hypervis_scaling, & - dcmip16_mu, ftype, theta_adv_form, prescribed_wind, moisture, & + dcmip16_mu, ftype, theta_adv_form, prescribed_wind, use_moisture, & disable_diagnostics, use_cpstar, transport_alg, & theta_hydrostatic_mode, test_case_name, dt_remap_factor, & dt_tracer_factor, scale_factor, laplacian_rigid_factor, & nsplit, pgrad_correction, dp3d_thresh, vtheta_thresh, & internal_diagnostics_level) bind(c) - use iso_c_binding, only: c_int, c_bool, c_double, c_ptr + use iso_c_binding, only: c_int, c_double, c_ptr ! ! Inputs ! @@ -29,8 +29,8 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, scale_factor, laplacian_rigid_factor, dp3d_thresh, vtheta_thresh integer(kind=c_int), intent(in) :: hypervis_order, hypervis_subcycle, hypervis_subcycle_tom integer(kind=c_int), intent(in) :: ftype, theta_adv_form - logical(kind=c_bool), intent(in) :: prescribed_wind, moisture, disable_diagnostics, use_cpstar - logical(kind=c_bool), intent(in) :: theta_hydrostatic_mode, pgrad_correction + integer(kind=c_int), intent(in) :: prescribed_wind, use_moisture, disable_diagnostics, use_cpstar + integer(kind=c_int), intent(in) :: theta_hydrostatic_mode, pgrad_correction type(c_ptr), intent(in) :: test_case_name end subroutine init_simulation_params_c @@ -138,11 +138,11 @@ end subroutine init_reference_element_c ! Create C++ functors subroutine init_functors_c (allocate_buffer) bind(c) - use iso_c_binding, only: c_bool + use iso_c_binding, only: c_int ! ! Inputs ! - logical(kind=c_bool), intent(in) :: allocate_buffer + integer(kind=c_int), intent(in) :: allocate_buffer end subroutine init_functors_c ! Initialize C++ boundary exchange structures From 0200065081910233ce6c861471b062d3e0608a2a Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:33:51 -0600 Subject: [PATCH 033/366] HOMME: some SYCL related mods in kokkos initialization and team policy defaults --- .../homme/src/share/cxx/ExecSpaceDefs.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index 8d496bff5d1..c9ca8a0ecd9 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -21,6 +21,10 @@ #include #endif +#ifdef KOKKOS_ENABLE_SYCL +#include +#endif + namespace Homme { // Since we're initializing from inside a Fortran code and don't have access to @@ -52,7 +56,16 @@ void initialize_kokkos () { // It isn't a big deal if we can't get the device count. nd = 1; } +#elif defined(KOKKOS_ENABLE_SYCL) + +//https://developer.codeplay.com/products/computecpp/ce/2.11.0/guides/sycl-for-cuda-developers/migrating-from-cuda-to-sycl + +//to make it build + int nd = 1; + #endif + + #ifdef HOMMEXX_ENABLE_GPU std::stringstream ss; ss << "--kokkos-num-devices=" << nd; @@ -117,6 +130,7 @@ team_num_threads_vectors_for_gpu ( assert(num_warps_total >= max_num_warps); assert(tp.max_threads_usable >= 1 && tp.max_vectors_usable >= 1); +#ifndef KOKKOS_ENABLE_SYCL int num_warps; if (tp.prefer_larger_team) { const int num_warps_usable = @@ -161,6 +175,9 @@ team_num_threads_vectors_for_gpu ( return std::make_pair( num_device_threads / num_vectors, num_vectors ); } +#else + return std::make_pair(4,16); +#endif } } // namespace Parallel From 5a43946dcf66085a54e64af76c8c5fade9a0e575 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:34:39 -0600 Subject: [PATCH 034/366] HOMME: do not add dependency on cprnc if building without PIO --- components/homme/test_execs/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/homme/test_execs/CMakeLists.txt b/components/homme/test_execs/CMakeLists.txt index a3113921b02..a007a5532b6 100644 --- a/components/homme/test_execs/CMakeLists.txt +++ b/components/homme/test_execs/CMakeLists.txt @@ -142,8 +142,11 @@ ADD_CUSTOM_TARGET(test-execs) ADD_CUSTOM_TARGET(check COMMAND ${CMAKE_CTEST_COMMAND} "--output-on-failure") +if(NOT BUILD_HOMME_WITHOUT_PIOLIBRARY) # Force cprnc to be built when make check is run ADD_DEPENDENCIES(check cprnc) +endif() + # Create a target for making the reference data ADD_CUSTOM_TARGET(baseline From 132b3a506b8a90274bcb9cc5f7c67cf2eafdfb20 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:30:50 -0600 Subject: [PATCH 035/366] HOMME: minor GPTL timing related mods * Disable timing in prim_main for the first two teps * Add some timers in EulerStepFunctor --- components/homme/src/prim_main.F90 | 7 ++++++- components/homme/src/share/cxx/EulerStepFunctorImpl.hpp | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/components/homme/src/prim_main.F90 b/components/homme/src/prim_main.F90 index bfbe57e8b31..d6901151d36 100644 --- a/components/homme/src/prim_main.F90 +++ b/components/homme/src/prim_main.F90 @@ -20,7 +20,7 @@ program prim_main use element_mod, only: element_t use common_io_mod, only: output_dir, infilenames use common_movie_mod, only: nextoutputstep - use perf_mod, only: t_initf, t_prf, t_finalizef, t_startf, t_stopf ! _EXTERNAL + use perf_mod, only: t_initf, t_prf, t_finalizef, t_startf, t_stopf, t_disablef, t_enablef ! _EXTERNAL use restart_io_mod , only: restartheader_t, writerestart use hybrid_mod, only: hybrid_create #if (defined MODEL_THETA_L && defined ARKODE) @@ -240,6 +240,11 @@ end subroutine finalize_kokkos_f90 nstep = nextoutputstep(tl) do while(tl%nstep= 2) call t_enablef() call t_startf('prim_run') call prim_run_subcycle(elem, hybrid,nets,nete, tstep, .false., tl, hvcoord,1) call t_stopf('prim_run') diff --git a/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp b/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp index f3029764dac..f87bb108beb 100644 --- a/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp +++ b/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp @@ -652,7 +652,10 @@ class EulerStepFunctorImpl { minmax_and_biharmonic(); } } + + GPTLstart("tl-at adv-n-limit"); advect_and_limit(); + GPTLstop("tl-at adv-n-limit"); exchange_qdp_dss_var(); } @@ -667,6 +670,7 @@ class EulerStepFunctorImpl { void run_tracer_phase (const KernelVariables& kv) const { compute_qtens(kv); kv.team_barrier(); + if (m_data.limiter_option == 8) { limiter_optim_iter_full(kv); kv.team_barrier(); @@ -674,6 +678,7 @@ class EulerStepFunctorImpl { limiter_clip_and_sum(kv); kv.team_barrier(); } + apply_spheremp(kv); } From 2e56ac054e317d047ccee5e37c40dbff2e9c7cdc Mon Sep 17 00:00:00 2001 From: noel Date: Sat, 7 Sep 2024 11:45:46 -0700 Subject: [PATCH 036/366] upgrade to intel/2023.2.0 on pm-cpu --- cime_config/machines/config_machines.xml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 6cd9d585a25..5b693e45526 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -223,8 +223,8 @@ - PrgEnv-intel/8.3.3 - intel/2023.1.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 @@ -239,13 +239,25 @@ cray-libsci/23.02.1.1 - + + craype-accel-host + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 + + + craype-accel-host craype/2.7.20 cray-mpich/8.1.25 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + + + cmake/3.24.3 evp-patch From 30b9f60a441723c13a6e111767b234f8eb482f73 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 9 Sep 2024 14:14:42 -0500 Subject: [PATCH 037/366] Hommexx: Isolate int-bool workaround to just the C++-F90 interface code. Also fix preqx's use of use_moisture. Remove MOIST-DRY enum to avoid confusion, since it's no longer used. --- .../homme/src/preqx_kokkos/cxx/CamForcing.cpp | 6 ++--- .../cxx/cxx_f90_interface_preqx.cpp | 4 +-- .../src/preqx_kokkos/cxx/prim_advance_exp.cpp | 2 +- components/homme/src/share/control_mod.F90 | 1 - components/homme/src/share/cxx/GllFvRemap.cpp | 6 ++--- components/homme/src/share/cxx/GllFvRemap.hpp | 2 +- .../homme/src/share/cxx/GllFvRemapImpl.cpp | 6 ++--- .../homme/src/share/cxx/GllFvRemapImpl.hpp | 5 ++-- .../homme/src/share/cxx/HommexxEnums.hpp | 5 ---- components/homme/src/share/gllfvremap_mod.F90 | 9 ++++--- components/homme/src/share/namelist_mod.F90 | 6 ----- .../theta-l_kokkos/cxx/EquationOfState.hpp | 4 +-- .../cxx/HyperviscosityFunctorImpl.cpp | 6 +---- .../cxx/HyperviscosityFunctorImpl.hpp | 2 +- .../src/theta-l_kokkos/prim_driver_mod.F90 | 25 +++++++++---------- .../thetal_kokkos_ut/forcing_ut.cpp | 8 +++--- .../thetal_kokkos_ut/gllfvremap_ut.cpp | 4 +-- 17 files changed, 42 insertions(+), 59 deletions(-) diff --git a/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp b/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp index 2b1e6514389..36ca5f4a95f 100644 --- a/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp @@ -51,7 +51,7 @@ void state_forcing( void tracer_forcing( const ExecViewUnmanaged &f_q, const HybridVCoord &hvcoord, const TimeLevel &tl, const int &num_q, - const MoistDry &moisture, const double &dt, + const bool &use_moisture, const double &dt, const ExecViewManaged &ps_v, const ExecViewManaged< Scalar * [Q_NUM_TIME_LEVELS][QSIZE_D][NP][NP][NUM_LEV]> &qdp, @@ -61,7 +61,7 @@ void tracer_forcing( const int np1 = tl.n0; const int np1_qdp = tl.n0_qdp; - if (moisture == MoistDry::MOIST) { + if (use_moisture) { // Remove the m_fq_ps_v buffer since it's not actually needed. // Instead apply the forcing to m_ps_v directly // Bonus - one less parallel reduce in dry cases! @@ -161,7 +161,7 @@ void apply_cam_forcing(const Real &dt) { tracers.fq = decltype(tracers.fq)("fq", elems.num_elems(),tracers.num_tracers()); } tracer_forcing(tracers.fq, hvcoord, tl, tracers.num_tracers(), - sim_params.moisture, dt, elems.m_state.m_ps_v, tracers.qdp, tracers.Q); + sim_params.use_moisture, dt, elems.m_state.m_ps_v, tracers.qdp, tracers.Q); GPTLstop("ApplyCAMForcing"); } diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index c75143a9836..b433a48c2ab 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -37,7 +37,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const int& time_step_type, const int& qsize, const int& state_frequency, const Real& nu, const Real& nu_p, const Real& nu_q, const Real& nu_s, const Real& nu_div, const Real& nu_top, const int& hypervis_order, const int& hypervis_subcycle, const double& hypervis_scaling, - const int& ftype, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, + const int& ftype, const bool& prescribed_wind, const bool& use_moisture, const bool& disable_diagnostics, const bool& use_cpstar, const int& transport_alg, const int& dt_remap_factor, const int& dt_tracer_factor, const double& scale_factor, const double& laplacian_rigid_factor) @@ -90,7 +90,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_subcycle = hypervis_subcycle; params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = disable_diagnostics; - params.moisture = (moisture ? MoistDry::MOIST : MoistDry::DRY); + params.use_moisture = use_moisture; params.use_cpstar = use_cpstar; params.transport_alg = transport_alg; // SphereOperators parameters; preqx supports only the sphere. diff --git a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp index f7c7600aab8..58e58f0160b 100644 --- a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp @@ -34,7 +34,7 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost // Determine the tracers time level tl.n0_qdp= -1; - if (params.moisture == MoistDry::MOIST) { + if (params.use_moisture) { tl.update_tracers_levels(params.qsplit); } diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 9c3c599b232..0e9494f5a6c 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -43,7 +43,6 @@ module control_mod ! flag used by preqx, theta-l and theta-c models ! should be renamed to "hydrostatic_mode" logical, public :: theta_hydrostatic_mode - integer, public :: theta_hydrostatic_mode_integer integer, public :: tstep_type= 5 ! preqx timestepping options diff --git a/components/homme/src/share/cxx/GllFvRemap.cpp b/components/homme/src/share/cxx/GllFvRemap.cpp index a8f564958d4..7b0400427f3 100644 --- a/components/homme/src/share/cxx/GllFvRemap.cpp +++ b/components/homme/src/share/cxx/GllFvRemap.cpp @@ -21,8 +21,8 @@ void init_gllfvremap_c (int nelemd, int np, int nf, int nf_max, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f) { auto& c = Context::singleton(); auto& g = c.get(); - g.init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, g2f_remapd, - f2g_remapd, D_f, Dinv_f); + const bool thm = static_cast(theta_hydrostatic_mode); + g.init_data(nf, nf_max, thm, fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f); } GllFvRemap::GllFvRemap () { @@ -52,7 +52,7 @@ void GllFvRemap::init_boundary_exchanges () { } void GllFvRemap -::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f) { m_impl->init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, diff --git a/components/homme/src/share/cxx/GllFvRemap.hpp b/components/homme/src/share/cxx/GllFvRemap.hpp index 2adff0aeaa9..7ebf5a82b71 100644 --- a/components/homme/src/share/cxx/GllFvRemap.hpp +++ b/components/homme/src/share/cxx/GllFvRemap.hpp @@ -40,7 +40,7 @@ class GllFvRemap { typedef Phys2T::const_type CPhys2T; typedef Phys3T::const_type CPhys3T; - void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index d4ab5c89f51..ea1a52f5efd 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -131,7 +131,7 @@ void GllFvRemapImpl::init_boundary_exchanges () { template using FV = Kokkos::View; void GllFvRemapImpl -::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r) { using Kokkos::create_mirror_view; @@ -395,7 +395,7 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c const auto hvcoord = m_hvcoord; const bool use_moisture = m_data.use_moisture; - const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; const bool want_dp_fv_out = dp_fv_out_ptr != nullptr; VPhys2T dp_fv_out; @@ -605,7 +605,7 @@ run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, const auto fT = m_forcing.m_ft; const auto hvcoord = m_hvcoord; const auto dp3d = m_state.m_dp3d; - const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; EquationOfState eos; eos.init(theta_hydrostatic_mode, hvcoord); ElementOps ops; ops.init(hvcoord); const auto tu_ne = m_tu_ne; diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index 7388fddb123..11738b2bf45 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -60,8 +60,7 @@ struct GllFvRemapImpl { struct Data { int nelemd, qsize, nf2, n_dss_fld; - bool use_moisture; - int theta_hydrostatic_mode; + bool use_moisture, theta_hydrostatic_mode; static constexpr int nbuf1 = 2, nbuf2 = 1; Buf1 buf1[nbuf1]; @@ -108,7 +107,7 @@ struct GllFvRemapImpl { void init_buffers(const FunctorsBuffersManager& fbm); void init_boundary_exchanges(); - void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r); diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 59c8f3c9652..06abbf35adb 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -47,11 +47,6 @@ enum class ForcingAlg : int { FORCING_2 = 2, // TODO: Rename FORCING_1 and FORCING_2 to something more descriptive }; -enum class MoistDry { - MOIST, - DRY -}; - enum class AdvectionForm { Conservative, NonConservative diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e927f04aba0..a5f9b3033c9 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -265,14 +265,14 @@ end subroutine gfr_init subroutine gfr_init_hxx() bind(c) #if KOKKOS_TARGET - use control_mod, only: theta_hydrostatic_mode_integer + use control_mod, only: theta_hydrostatic_mode use iso_c_binding, only: c_int interface - subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode_integer, & + subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode, & fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f) bind(c) use iso_c_binding, only: c_int, c_double integer (c_int), value, intent(in) :: nelemd, np, nf, nf_max - integer (c_int), value, intent(in) :: theta_hydrostatic_mode_integer + integer (c_int), value, intent(in) :: theta_hydrostatic_mode real (c_double), dimension(nf*nf,nelemd), intent(in) :: fv_metdet real (c_double), dimension(np,np,nf_max*nf_max), intent(in) :: g2f_remapd real (c_double), dimension(nf_max*nf_max,np,np), intent(in) :: f2g_remapd @@ -280,7 +280,8 @@ subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode_inte end subroutine init_gllfvremap_c end interface integer (c_int) :: thm - thm = theta_hydrostatic_mode_integer + thm = 0 + if (theta_hydrostatic_mode) thm = 1 call init_gllfvremap_c(nelemd, np, gfr%nphys, nphys_max, thm, & gfr%fv_metdet, gfr%g2f_remapd, gfr%f2g_remapd, gfr%D_f, gfr%Dinv_f) #endif diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index a3edaa07e23..1d47090182b 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -41,7 +41,6 @@ module namelist_mod runtype, & integration, & ! integration method theta_hydrostatic_mode, & - theta_hydrostatic_mode_integer, & transport_alg , & ! SE Eulerian, classical SL, cell-integrated SL semi_lagrange_cdr_alg, & ! see control_mod for semi_lagrange_* descriptions semi_lagrange_cdr_check, & @@ -453,10 +452,8 @@ subroutine readnl(par) planar_slice = .false. theta_hydrostatic_mode = .true. ! for preqx, this must be .true. - theta_hydrostatic_mode_integer = 1 ! for preqx, this must be .true. #if ( defined MODEL_THETA_C || defined MODEL_THETA_L ) theta_hydrostatic_mode = .false. ! default NH - theta_hydrostatic_mode_integer = 0 ! default NH #endif @@ -853,10 +850,7 @@ subroutine readnl(par) call MPI_bcast(case_planar_bubble,1,MPIlogical_t,par%root,par%comm,ierr) #endif -if(theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 1 -if(.not. theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 0 call MPI_bcast(theta_hydrostatic_mode ,1,MPIlogical_t,par%root,par%comm,ierr) - call MPI_bcast(theta_hydrostatic_mode_integer ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(transport_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_check ,1,MPIlogical_t,par%root,par%comm,ierr) diff --git a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp index a50a28d58f5..dd97720f1be 100644 --- a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp @@ -23,7 +23,7 @@ class EquationOfState { EquationOfState () = default; - void init (const int theta_hydrostatic_mode, + void init (const bool theta_hydrostatic_mode, const HybridVCoord& hvcoord) { m_theta_hydrostatic_mode = theta_hydrostatic_mode; m_hvcoord = hvcoord; @@ -250,7 +250,7 @@ class EquationOfState { public: - int m_theta_hydrostatic_mode; + bool m_theta_hydrostatic_mode; HybridVCoord m_hvcoord; }; diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index 55792051d33..d160e114475 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -120,11 +120,7 @@ void HyperviscosityFunctorImpl::init_params(const SimulationParams& params) #ifdef HOMMEXX_BFB_TESTING m_process_nh_vars = 1; #else - if (params.theta_hydrostatic_mode){ - m_process_nh_vars = 0; - }else{ - m_process_nh_vars = 1; - } + m_process_nh_vars = not params.theta_hydrostatic_mode; #endif } diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp index 993d525422f..a55ecbb365f 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp @@ -397,7 +397,7 @@ class HyperviscosityFunctorImpl Buffers m_buffers; HybridVCoord m_hvcoord; - int m_process_nh_vars; + bool m_process_nh_vars; // Policies Kokkos::TeamPolicy m_policy_update_states; diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 262ba19f4b7..eae8544ca86 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -103,12 +103,12 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) ! Fill the simulation params structures in C++ test_name = TRIM(test_case) // C_NULL_CHAR - if (disable_diagnostics) disable_diagnostics_int=1 - if (.not.disable_diagnostics) disable_diagnostics_int=0 - if (use_moisture) use_moisture_int=1 - if (.not.use_moisture) use_moisture_int=0 - if(theta_hydrostatic_mode) theta_hydrostatic_mode_int=1 - if(.not.theta_hydrostatic_mode) theta_hydrostatic_mode_int=0 + disable_diagnostics_int = 0 + if (disable_diagnostics) disable_diagnostics_int = 1 + use_moisture_int = 0 + if (use_moisture) use_moisture_int = 1 + theta_hydrostatic_mode_int = 0 + if (theta_hydrostatic_mode) theta_hydrostatic_mode_int = 1 call init_simulation_params_c (vert_remap_q_alg, limiter_option, rsplit, qsplit, tstep_type, & qsize, statefreq, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & @@ -358,17 +358,16 @@ subroutine prim_init_kokkos_functors (allocate_buffer) ! ! Optional Input ! - integer, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally - integer(kind=c_int) :: dummy + logical, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + integer(kind=c_int) :: ab ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, ! let Homme internally allocate buffers + ab = 1 if (present(allocate_buffer)) then - call init_functors_c (allocate_buffer) - else - dummy=1; - call init_functors_c (dummy) - endif + if (.not. allocate_buffer) ab = 0 + end if + call init_functors_c (ab) ! Initialize boundary exchange structure in C++ call init_boundary_exchanges_c () diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index 5e4c51c7ca1..fb301166f42 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -160,8 +160,8 @@ TEST_CASE("forcing", "forcing") { std::cout << "Testing tracers forcing.\n"; for (const bool hydrostatic : {true,false}) { std::cout << " -> hydrostatic mode: " << (hydrostatic ? "true" : "false") << "\n"; - for (const MoistDry moisture : {MoistDry::DRY,MoistDry::MOIST}) { - std::cout << " -> moisture: " << (moisture==MoistDry::MOIST ? "moist" : "dry") << "\n"; + for (const bool use_moisture: {false,true}) { + std::cout << " -> moisture: " << (use_moisture ? "moist" : "dry") << "\n"; for (const bool adjustment : {true,false}) { std::cout << " -> adjustment: " << (adjustment ? "true" : "false") << "\n"; @@ -200,8 +200,8 @@ TEST_CASE("forcing", "forcing") { ff.init_buffers(fbm); // Run tracers forcing (cxx and f90) - ff.tracers_forcing(dt,np1,np1_qdp,adjustment,moisture); - tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,moisture==MoistDry::MOIST,adjustment); + ff.tracers_forcing(dt,np1,np1_qdp,adjustment,use_moisture); + tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,use_moisture,adjustment); // Compare answers Kokkos::deep_copy(h_dp,state.m_dp3d); diff --git a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp index 0f14b0c3e55..cf9db941ea1 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp @@ -183,7 +183,7 @@ struct Session { p.qsize = qsize; p.hypervis_scaling = 0; p.transport_alg = 0; - p.moisture = MoistDry::MOIST; + p.use_moisture = true; p.theta_hydrostatic_mode = false; p.scale_factor = is_sphere ? PhysicalConstants::rearth0 : 1; p.laplacian_rigid_factor = is_sphere ? 1/p.scale_factor : 0; @@ -725,7 +725,7 @@ static void test_get_temperature (Session& s) { const auto& sp = c.get(); EquationOfState eos; eos.init(theta_hydrostatic_mode, s.h); ElementOps ops; ops.init(s.h); - const bool use_moisture = sp.moisture == MoistDry::MOIST; + const bool use_moisture = sp.use_moisture; const auto state = c.get(); const auto tracers = c.get(); const auto dp3d = state.m_dp3d; From d1c60b4ff8e12cd3e88c31890cdc22816ac593dd Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Mon, 9 Sep 2024 15:49:31 -0500 Subject: [PATCH 038/366] move ww3 build to live in the run directory --- components/ww3/cime_config/buildlib_cmake | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/components/ww3/cime_config/buildlib_cmake b/components/ww3/cime_config/buildlib_cmake index 4f15ea2d58e..9df52bc35c6 100755 --- a/components/ww3/cime_config/buildlib_cmake +++ b/components/ww3/cime_config/buildlib_cmake @@ -43,14 +43,23 @@ def buildlib(bldroot, installpath, case): # Define WW3 repository directories repodir = "{}/components/ww3/src".format(srcroot) modeldir = "{}/WW3/model".format(repodir) + builddir = "{}/wav".format(exeroot) # TODO: these work dirs will have to be changed to live in the binary/build area. # Achieving this will probably require a significant refactor of the ww3 infrastructure. # Doing this stuff in-source not only clutters the repo, but also introduces potential race # conditions if we were to try to build multiple ww3 cases simultaneously. - bindir = "{}/bin".format(modeldir) - exedir = "{}/exe".format(modeldir) - tmpdir = "{}/tmp".format(modeldir) + bindir1 = "{}/bin".format(modeldir) + bindir = "{}/wav/bin".format(exeroot) + shutil.copytree(bindir1, bindir) + auxdir1 = "{}/aux".format(modeldir) + auxdir = "{}/wav/aux".format(exeroot) + shutil.copytree(auxdir1, auxdir) + ftndir1 = "{}/ftn".format(modeldir) + ftndir = "{}/wav/ftn".format(exeroot) + shutil.copytree(ftndir1, ftndir) + + tmpdir = "{}/wav/tmp".format(exeroot) # Run w3_setup to create wwatch3.env file env_file = os.path.join(bindir, "wwatch3.env") @@ -77,7 +86,7 @@ def buildlib(bldroot, installpath, case): y """.format(sf90, scc, tmpdir)) - run_bld_cmd_ensure_logging("./w3_setup {} -s E3SM < w3_setup.inp".format(modeldir), logger, from_dir=bindir) + run_bld_cmd_ensure_logging("./w3_setup {} -s E3SM < w3_setup.inp".format(builddir), logger, from_dir=bindir) os.remove(inp_file) # Generate pre-processed WW3 source code @@ -85,7 +94,7 @@ y for exe in ww3_exe: run_bld_cmd_ensure_logging("./w3_source {}".format(exe), logger, from_dir=bindir) for exe in ww3_exe: - tarfile = "{}/work/{}.tar.gz".format(modeldir,exe) + tarfile = "{}/work/{}.tar.gz".format(builddir,exe) shutil.move(tarfile, tmpdir) run_bld_cmd_ensure_logging("tar -xzvf {}.tar.gz".format(exe), logger, from_dir=tmpdir) run_bld_cmd_ensure_logging("rm ww3_shel.F90", logger, from_dir=tmpdir) From 674d6adaa9062df8c549d598218008b98b7c5996 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Tue, 10 Sep 2024 11:21:19 -0500 Subject: [PATCH 039/366] Add grid support for ERA5r025 configurations --- cime_config/config_grids.xml | 67 +++++++++++++++++++ .../elm/cime_config/config_compsets.xml | 5 ++ 2 files changed, 72 insertions(+) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 0f1c3512ec8..3e47fd4d754 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -1979,6 +1979,26 @@ IcoswISC30E3r5 + + ERA5r025 + r05 + IcoswISC30E3r5 + r05 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + + + ERA5r025 + r025 + IcoswISC30E3r5 + r025 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + ne120np4.pg2 r0125 @@ -2801,6 +2821,14 @@ TL319 is JRA lat/lon grid: + + 1440 + 721 + $DIN_LOC_ROOT/share/domains/domain.lnd.ERA5r025_IcoswISC30E3r5.240903.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ERA5r025_IcoswISC30E3r5.240903.nc + ERA5r025 is the lat/lon cap grid used by ERA5 data: + + @@ -4732,6 +4760,38 @@ cpl/gridmaps/TL319/map_r05_to_TL319_traave.20240212.nc + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5-nomask_trbilin.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5_esmfpatch.20240903.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ERA5r025_traave.20240903.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ERA5r025_traave.20240903.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_traave.20240905.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_esmfbilin.20240907.nc + cpl/gridmaps/r05/map_r05_to_ERA5r025_traave.20240905.nc + cpl/gridmaps/r05/map_r05_to_ERA5r025_traave.20240905.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_traave.20240905.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_esmfbilin.20240907.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_trbilin.20240903.nc + cpl/gridmaps/r025/map_r025_to_ERA5r025_traave.20240903.nc + cpl/gridmaps/r025/map_r025_to_ERA5r025_traave.20240903.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_trbilin.20240903.nc + + cpl/cpl6/map_T31_to_gx3v7_aave_da_090903.nc cpl/cpl6/map_T31_to_gx3v7_patch_090903.nc @@ -5635,6 +5695,13 @@ cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r05_traave.20240403.nc + + cpl/gridmaps/r025/map_r025_to_gis1to10kmR2_traave.20240903.nc + cpl/gridmaps/r025/map_r025_to_gis1to10kmR2_trbilin.20240903.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r025_traave.20240903.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r025_traave.20240903.nc + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_traave.20240403.nc cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_trbilin.20240403.nc diff --git a/components/elm/cime_config/config_compsets.xml b/components/elm/cime_config/config_compsets.xml index 5eed24e49da..4afbdfbf3a3 100644 --- a/components/elm/cime_config/config_compsets.xml +++ b/components/elm/cime_config/config_compsets.xml @@ -972,6 +972,11 @@ 2000_DATM%QIA_ELM%SP_SICE_SOCN_MOSART_MALI_SWAV + + IGERA5ELM_MLI + 2000_DATM%ERA5_ELM%SP_SICE_SOCN_MOSART_MALI_SWAV + + I1PTELM 2000_DATM%1PT_ELM%SP_SICE_SOCN_MOSART_SGLC_SWAV From 26cb157e61f35ecfaa8e2f859b2065a09be38dbc Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Tue, 10 Sep 2024 12:14:31 -0500 Subject: [PATCH 040/366] Add some r025 files used by ERA5r025_r025_IcoswISC30E3r5_gis1to10r02 --- components/elm/bld/namelist_files/namelist_defaults.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index e76a5647b4e..cc1bd512e69 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -613,6 +613,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/griddata/glcmaskdata_0.9x1.25_60S.nc lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc +lnd/clm2/griddata/glcmaskdata_r025_GIS_AIS.20240910.nc lnd/clm2/griddata/glcmaskdata_r0125_GIS_AIS.210407.nc lnd/clm2/griddata/glcmaskdata_ne30pg2_GIS_AIS.210407.nc @@ -623,6 +624,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/griddata/topodata_48x96_USGS_070110.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c200609.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c211019.nc +lnd/clm2/surfdata_map/surfdata_0.25x0.25_simyr1850_c240125.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr1850_c190730.nc lnd/clm2/surfdata_map/surfdata_ne30pg2_simyr1850_c210402.nc From f38fe0e46ab0a225a126f793e5fbcee910dd7a78 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 10 Sep 2024 15:56:02 -0500 Subject: [PATCH 041/366] big flow error compute rof lnd interaction only if rof_c2_lnd is true we were computing an unnecessary map always even if rof_c2_lnd is false (for most coupled cases it is true) --- driver-moab/main/prep_lnd_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index 44ae5490e2c..0272e835ea0 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -222,7 +222,6 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln call seq_map_init_rcfile(mapper_Fr2l, rof(1), lnd(1), & 'seq_maps.rc','rof2lnd_fmapname:','rof2lnd_fmaptype:',samegrid_lr, & string='mapper_Fr2l initialization',esmf_map=esmf_map_flag) - end if ! symmetric of l2r, from prep_rof #ifdef HAVE_MOAB ! Call moab intx only if land and river are init in moab @@ -381,6 +380,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln end if ! if ((mbrxid .ge. 0) .and. (mblxid .ge. 0)) ! endif HAVE_MOAB #endif + end if call shr_sys_flush(logunit) if (atm_c2_lnd) then From 1892887ede89b000b3165ea5f76a05c5e1a3dbe2 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 10 Sep 2024 16:15:26 -0500 Subject: [PATCH 042/366] MPI group needs to be retrieved before use in case rof c2 lnd, we were not retrieving the the MPI group needed for comm graph computation, for same grid path we were retrieving it for intx only how come we never had samegrid land rof before ? --- driver-moab/main/prep_rof_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index 59cc61c5889..a6a9184aefb 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -327,6 +327,7 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) write(logunit,*) subname,' error in defining tags for seq_flds_a2x_fields on rof cpl' call shr_sys_abort(subname//' ERROR in defining tags for seq_flds_a2x_fields on rof cpl') endif + call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) if (samegrid_lr) then ! the same mesh , lnd and rof use the same dofs, but restricted ! we do not compute intersection, so we will have to just send data from lnd to rof and viceversa, by GLOBAL_ID matching @@ -363,7 +364,6 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) ! we also need to compute the comm graph for the second hop, from the lnd on coupler to the ! lnd for the intx lnd-rof context (coverage) ! - call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) type1 = 3 ! land is FV now on coupler side type2 = 3; From e4b86b6e707aa57589e4361001f1540ca4b8542b Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 10 Sep 2024 18:35:51 -0500 Subject: [PATCH 043/366] reviews from Vijay --- .../data_comps/dlnd/src/dlnd_comp_mod.F90 | 4 +- driver-moab/main/prep_lnd_mod.F90 | 8 +-- driver-moab/main/prep_ocn_mod.F90 | 66 +++++++++---------- driver-moab/main/prep_rof_mod.F90 | 32 ++++----- 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index 7768b2dfe81..4d76c0fe1a1 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -148,10 +148,10 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & #ifdef HAVE_MOAB character*400 tagname real(R8) latv, lonv - integer iv, tagindex, ilat, ilon !, arrsize, nfields ! ierr is already defined as integer above + integer iv, tagindex, ilat, ilon real(R8), allocatable, target :: data(:) integer(IN), pointer :: idata(:) ! temporary - real(r8), dimension(:), allocatable :: moab_vert_coords ! temporary + real(R8), dimension(:), allocatable :: moab_vert_coords ! temporary #ifdef MOABDEBUG character*100 outfile, wopts #endif diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index 0272e835ea0..7bdb12376a3 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -1,6 +1,6 @@ module prep_lnd_mod - use shr_kind_mod , only: r8 => SHR_KIND_R8 + use shr_kind_mod , only: R8 => SHR_KIND_R8 use shr_kind_mod , only: cs => SHR_KIND_CS use shr_kind_mod , only: cl => SHR_KIND_CL use shr_kind_mod , only: cxx => SHR_KIND_CXX @@ -162,7 +162,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln integer nrflds ! number of rof fields projected on land integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 #endif character(*), parameter :: subname = '(prep_lnd_init)' @@ -369,7 +369,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mblxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on land' @@ -693,7 +693,7 @@ subroutine prep_lnd_mrg_moab (infodata) #endif #ifdef MOABCOMP character(CXX) :: tagname, mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list, ent_type type(mct_string) :: mctOStr ! diff --git a/driver-moab/main/prep_ocn_mod.F90 b/driver-moab/main/prep_ocn_mod.F90 index 44ec0c6a3c7..b13cce62f95 100644 --- a/driver-moab/main/prep_ocn_mod.F90 +++ b/driver-moab/main/prep_ocn_mod.F90 @@ -1,6 +1,6 @@ module prep_ocn_mod - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: CX => shr_kind_CX, CXX => shr_kind_CXX @@ -137,7 +137,7 @@ module prep_ocn_mod integer , target :: x2oacc_ox_cnt ! x2oacc_ox: number of time samples accumulated ! accumulation variables for moab data - real (kind=r8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array + real (kind=R8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array integer , target :: x2oacc_om_cnt ! x2oacc_ox: number of time samples accumulated, in moab array integer :: arrSize_x2o_om ! this will be a module variable, size moabLocal_size * nof @@ -154,20 +154,20 @@ module prep_ocn_mod !================================================================================================ - real (kind=r8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them + real (kind=R8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them ! they were init with ! character(*),parameter :: fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2o_om (:,:) - real (kind=r8) , allocatable, private :: a2x_om (:,:) - real (kind=r8) , allocatable, private :: i2x_om (:,:) - real (kind=r8) , allocatable, private :: r2x_om (:,:) - real (kind=r8) , allocatable, private :: xao_om (:,:) + real (kind=R8) , allocatable, private :: x2o_om (:,:) + real (kind=R8) , allocatable, private :: a2x_om (:,:) + real (kind=R8) , allocatable, private :: i2x_om (:,:) + real (kind=R8) , allocatable, private :: r2x_om (:,:) + real (kind=R8) , allocatable, private :: xao_om (:,:) ! this will be constructed first time, and be used to copy fields for shared indices ! between xao and x2o character(CXX) :: shared_fields_xao_x2o ! will need some array to hold the data for copying - real(r8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize + real(R8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize integer :: size_of_shared_values logical :: iamin_CPLALLICEID ! pe associated with CPLALLICEID @@ -268,7 +268,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags integer noflds ! used for number of fields in allocating moab accumulated array x2oacc_om - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 !--------------------------------------------------------------- @@ -786,7 +786,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mboxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on ocn' @@ -1095,7 +1095,7 @@ subroutine prep_ocn_mrg(infodata, fractions_ox, xao_ox, timer_mrg) ! ! Local Variables integer :: eii, ewi, egi, eoi, eai, eri, exi, efi, emi - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor type(mct_avect), pointer :: x2o_ox integer :: cnt character(*), parameter :: subname = '(prep_ocn_mrg)' @@ -1177,7 +1177,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) !--------------------------------------------------------------- - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor ! will build x2o_om , similar to x2o_ox ! no averages, just one ocn instance @@ -1187,11 +1187,11 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) integer :: kof,kif integer :: lsize, arrsize ! for double arrays integer , save :: noflds,naflds,niflds,nrflds,nxflds! ,ngflds,nwflds, no glacier or wave model - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -1289,7 +1289,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) #endif #ifdef MOABCOMP character(CXX) :: mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1345,7 +1345,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) allocate(a2x_om (lsize, naflds)) allocate(i2x_om (lsize, niflds)) allocate(r2x_om (lsize, nrflds)) - r2x_om = 0._r8 ! should we zero out all of them ? + r2x_om = 0._R8 ! should we zero out all of them ? allocate(xao_om (lsize, nxflds)) ! allocate fractions too ! use the fraclist fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' @@ -1769,7 +1769,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) ! fo_kif_ifrac(n) ! fractions_o%rAttr(kif,n) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -1777,7 +1777,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifracr = fractions_om(n,kir) ! fo_kir_ifrad(n) ! fractions_o%rAttr(kir,n) afracr = fractions_om(n,kor) ! fo_kor_ofrad(n) ! fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -1913,7 +1913,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) !fo_kif_ifrac(n) ! fractions_o%rAttr(kif) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2046,7 +2046,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa !----------------------------------------------------------------------- ! ! Arguments - real(r8) , intent(in) :: flux_epbalfact + real(R8) , intent(in) :: flux_epbalfact type(mct_aVect), intent(in) :: a2x_o type(mct_aVect), intent(in) :: i2x_o type(mct_aVect), intent(in) :: r2x_o @@ -2061,11 +2061,11 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa integer :: kof,kif integer :: lsize integer :: noflds,naflds,niflds,nrflds,nwflds,nxflds,ngflds - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -2584,7 +2584,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2592,7 +2592,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifracr = fractions_o%rAttr(kir,n) afracr = fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -2738,7 +2738,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -3068,7 +3068,7 @@ function prep_ocn_get_mapper_Sw2o() prep_ocn_get_mapper_Sw2o => mapper_Sw2o end function prep_ocn_get_mapper_Sw2o function prep_ocn_get_x2oacc_om() - real(r8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om + real(R8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om prep_ocn_get_x2oacc_om => x2oacc_om end function prep_ocn_get_x2oacc_om function prep_ocn_get_x2oacc_om_cnt() diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index a6a9184aefb..83a8e331e91 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -1,7 +1,7 @@ module prep_rof_mod #include "shr_assert.h" - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: cxx => SHR_KIND_CXX @@ -112,22 +112,22 @@ module prep_rof_mod ! accumulation variables over moab fields character(CXX) :: sharedFieldsLndRof ! used in moab to define l2racc_lm - real (kind=r8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes - real (kind=r8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module + real (kind=R8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes + real (kind=R8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module integer , target :: l2racc_lm_cnt ! l2racc_lm: number of time samples accumulated integer :: nfields_sh_lr ! number of fields in sharedFieldsLndRof integer :: lsize_lm ! size of land in moab, local character(CXX) :: sharedFieldsAtmRof ! used in moab to define a2racc_am - real (kind=r8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes - real (kind=r8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module + real (kind=R8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes + real (kind=R8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module integer , target :: a2racc_am_cnt ! a2racc_am: number of time samples accumulated integer :: nfields_sh_ar ! number of fields in sharedFieldsAtmRof integer :: lsize_am ! size of atm in moab, local character(CXX) :: sharedFieldsOcnRof ! used in moab to define o2racc_om - real (kind=r8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes - real (kind=r8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof + real (kind=R8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes + real (kind=R8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof integer , target :: o2racc_om_cnt ! o2racc_om: number of time samples accumulated integer :: nfields_sh_or ! number of fields in sharedFieldsOcnRof integer :: lsize_om ! size of ocn in moab, local @@ -145,12 +145,12 @@ module prep_rof_mod logical :: samegrid_al ! samegrid atm and lnd ! moab stuff - real (kind=r8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them + real (kind=R8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them ! they were init with ! character(*),parameter :: fraclist_r = 'lfrac:lfrin:rfrac' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2r_rm (:,:) ! result of merge - real (kind=r8) , allocatable, private :: a2x_rm (:,:) - real (kind=r8) , allocatable, private :: l2x_rm (:,:) + real (kind=R8) , allocatable, private :: x2r_rm (:,:) ! result of merge + real (kind=R8) , allocatable, private :: a2x_rm (:,:) + real (kind=R8) , allocatable, private :: l2x_rm (:,:) !================================================================================================ @@ -1223,7 +1223,7 @@ subroutine prep_rof_merge(l2x_r, a2x_r, fractions_r, x2r_r, cime_model,o2x_r) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1529,7 +1529,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1548,7 +1548,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) character*32 :: outfile, wopts, lnum #endif #ifdef MOABCOMP - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1995,7 +1995,7 @@ function prep_rof_get_o2racc_om_cnt() end function prep_rof_get_o2racc_om_cnt function prep_rof_get_o2racc_om() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om prep_rof_get_o2racc_om => o2racc_om end function prep_rof_get_o2racc_om @@ -2011,7 +2011,7 @@ function prep_rof_get_l2racc_lm_cnt() end function prep_rof_get_l2racc_lm_cnt function prep_rof_get_l2racc_lm() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm prep_rof_get_l2racc_lm => l2racc_lm end function prep_rof_get_l2racc_lm From 2faa92748a137729e3269cb399b2fafe258588fb Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Fri, 12 Jul 2024 13:53:00 -0400 Subject: [PATCH 044/366] Added downscaling code for coupler bypass --- .../elm/src/cpl/lnd_downscale_atm_forcing.F90 | 526 ++++++++++++++++++ components/elm/src/cpl/lnd_import_export.F90 | 90 +-- components/elm/src/main/atm2lndType.F90 | 5 + 3 files changed, 584 insertions(+), 37 deletions(-) diff --git a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 index 99b03a2e25a..8c003bbf5e9 100644 --- a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 +++ b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 @@ -35,10 +35,13 @@ module lnd_downscale_atm_forcing ! ! !PUBLIC MEMBER FUNCTIONS: public :: downscale_atm_forcing_to_topounit ! Calls downscaling subroutines of forcing fields from gridcell to topounit + public :: downscale_atm_forcing_to_topounit_cpl_bypass ! Calls downscaling subroutines of forcing fields from gridcell to topounit, for use within the CPL_BYPASS code ! ! !PRIVATE MEMBER FUNCTIONS: private :: downscale_atm_state_to_topounit ! Downscale atmosperic state fields from gridcell to topounit + private :: downscale_atm_state_to_topounit_cpl_bypass ! Downscale atmosperic state fields from gridcell to topounit, for cpl_bypass code private :: downscale_longwave_to_topounit ! Downscale longwave radiation field from gridcell to topounit + private :: downscale_longwave_to_topounit_cpl_bypass ! Downscale longwave radiation field from gridcell to topounit, for cpl_bypass code private :: downscale_precip_to_topounit_FNM ! Downscale precipitation field from gridcell to topounit using Froude number method (FNM) private :: downscale_precip_to_topounit_ERMM ! Downscale precipitation field from gridcell to topounit using elevation ration with maximum elevation method (ERMM) private :: build_normalization ! Compute normalization factors so that downscaled fields are conservative @@ -787,4 +790,527 @@ subroutine build_normalization(orig_field, sum_field, sum_wts, norms) end subroutine build_normalization + subroutine downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm_vars) + ! + ! !DESCRIPTION: + ! Downscale fields from gridcell to topounit + ! + ! Downscaling is done over topounits if the number of topounits > 1. + ! + ! !USES: + use elm_time_manager, only : get_nstep + use elm_varcon , only : rair, cpair, grav, lapse_glcmec + use elm_varcon , only : glcmec_rain_snow_threshold, o2_molar_const + use shr_const_mod , only : SHR_CONST_TKFRZ + use landunit_varcon , only : istice_mec + use elm_varctl , only : glcmec_downscale_rain_snow_convert + use domainMod , only : ldomain + use QsatMod , only : Qsat + use FrictionVelocityMod, only: atm_gustiness + ! + ! !ARGUMENTS: + integer , intent(in) :: g + real(r8) , intent(in) :: x2l(:,:) + type(atm2lnd_type) , intent(in) :: atm2atm_vars + type(lnd2atm_type) , intent(in) :: lnd2atm_vars + + ! + ! !LOCAL VARIABLES: + integer :: t, l, c, fc, t2 ! indices + integer :: clo, cc + integer :: numt_pg ! Number of topounits per grid + + ! temporaries for topo downscaling + real(r8) :: rain_g, snow_g + real(r8) :: mxElv ! Maximum elevation value per grid + real(r8) :: uovern_t ! Froude Number + real(r8) :: grdElv ! Grid elevation + real(r8) :: topoElv ! Topounit elevation + real(r8) :: max_tpuElv ! Maximum topounit elevation for calculating elevation range + real(r8) :: min_tpuElv ! Minimum topounit elevation for calculating elevation range + real(r8) :: crnt_temp_t ! Current downscaled topounit temperature + real(r8) :: temp_r ! Temporary topounit rainfall + real(r8) :: temp_s ! Temporary topounit snowfall + real(r8) :: t_th ! Temperature threshold for snowfall + real(r8) :: Ta_th1 ! Temperature at 0.5 Celsius in K 273.65 + real(r8) :: Ta_th2 ! Temperature at 2.0 Celsius in K 275.15 + real(r8) :: Ta_th3 ! Temperature at 2.5 Celsius in K 275.65 + real(r8) :: tmp_Snow_frc ! Current snow fraction + + real(r8) :: e + real(r8) :: qvsat + + real(r8) :: sum_qbot_g ! weighted sum of column-level lwrad + real(r8) :: sum_wtsq_g ! sum of weights that contribute to sum_lwrad_g + real(r8) :: qbot_norm_g ! normalization factors + real(r8) :: sum_lwrad_g ! weighted sum of column-level lwrad + real(r8) :: sum_wtslw_g ! sum of weights that contribute to sum_lwrad_g + real(r8) :: lwrad_norm_g ! normalization factors + real(r8) :: esatw ! saturation vapor pressure over water (Pa) + real(r8) :: esati ! saturation vapor pressure over ice (Pa) + real(r8) :: a0,a1,a2,a3,a4,a5,a6 ! coefficients for esat over water + real(r8) :: b0,b1,b2,b3,b4,b5,b6 ! coefficients for esat over ice + real(r8) :: tdc, temp ! Kelvins to Celcius function and its input + real(r8) :: vp ! water vapor pressure (Pa) + + real(r8), allocatable :: deltaRain(:) ! Deviation of subgrid rain from grid rain + real(r8), allocatable :: deltaSnow(:) ! Deviation of subgrid snow from grid snow + real(r8) :: deltaR ! Temporary deltaRain + real(r8) :: deltaS ! Temporary deltaSnow + real(r8) :: sum_of_hrise ! Sum of height rise of air parcel of all subgrids of a grid + real(r8) :: hrise ! Temporary height rise + real(r8) :: elvrnge ! Elevation range between lowest and highest tpu + real(r8) :: ave_elv + integer :: elv_flag ! Elevation flag to trac grids with +ve grid elevation and -ve tpu elevation + + integer :: uaflag = 0 + integer :: precip_dwn = 0 ! Used to turn on/off the downscaling of precipitation 0 = on; 1 = off + integer :: other_forcing_dwn = 0 ! Used to turn on/off the downscaling of other forcing 0 = on; 1 = off + + character(len=*), parameter :: subname = 'downscale_atm_forcing_to_topounit' + !---------------------------------------------------------------------------------------- + + ! Constants to compute vapor pressure + parameter (a0=6.107799961_r8 , a1=4.436518521e-01_r8, & + a2=1.428945805e-02_r8, a3=2.650648471e-04_r8, & + a4=3.031240396e-06_r8, a5=2.034080948e-08_r8, & + a6=6.136820929e-11_r8) + + parameter (b0=6.109177956_r8 , b1=5.034698970e-01_r8, & + b2=1.886013408e-02_r8, b3=4.176223716e-04_r8, & + b4=5.824720280e-06_r8, b5=4.838803174e-08_r8, & + b6=1.838826904e-10_r8) + + ! + ! function declarations + ! + tdc(temp) = min( 50._r8, max(-50._r8,(temp-SHR_CONST_TKFRZ)) ) ! Taken from lnd_import_export.F90 + esatw(temp) = 100._r8*(a0+temp*(a1+temp*(a2+temp*(a3+temp*(a4+temp*(a5+temp*a6)))))) ! Taken from lnd_import_export.F90 + esati(temp) = 100._r8*(b0+temp*(b1+temp*(b2+temp*(b3+temp*(b4+temp*(b5+temp*b6)))))) ! Taken from lnd_import_export.F90 + !----------------------------------------------------------------------- + ! Get required inputs + numt_pg = grc_pp%ntopounits(g) ! Number of topounits per grid + grdElv = grc_pp%elevation(g) ! Grid level sfc elevation + mxElv = grc_pp%MaxElevation(g) ! Maximum src elevation per grid obtained from the highest elevation topounit + uovern_t = atm2lnd_vars%forc_uovern(g) ! Froude Number + + snow_g = atm2lnd_vars%forc_snow_not_downscaled_grc(g) + rain_g = atm2lnd_vars%forc_rain_not_downscaled_grc(g) + + sum_qbot_g = 0._r8 + sum_wtsq_g = 0._r8 + sum_lwrad_g = 0._r8 + sum_wtslw_g = 0._r8 + + sum_of_hrise = 0._r8 + t_th = 273.15_r8 ! Freezing temperature in K + Ta_th1 = 273.65_r8 ! Lowest threshold for snow calculation + Ta_th2 = 275.15_r8 ! Middle threshold for rain/snow partitioning + Ta_th3 = 275.65_r8 ! Highest threshold for rain/snow partitioning + tmp_Snow_frc = 0._r8 ! Snow fraction + elv_flag = 0 + elvrnge = 0._r8 + ave_elv = 0._r8 + max_tpuElv = 0._r8 + min_tpuElv = 0._r8 + if (numt_pg > 1) then !downscaling is done only if a grid has more than 1 topounits + if (precip_downscaling_method == 'FNM') then + allocate(deltaRain(numt_pg)) + deltaRain(:) = 0._r8 + allocate(deltaSnow(numt_pg)) + deltaSnow(:) = 0._r8 + hrise = 0._r8 + end if + + ! calculate elevation range and track grids with +ve elevation but have -ve tpu elevation + max_tpuElv = -100000._r8 + min_tpuElv = 100000._r8 + do t = grc_pp%topi(g), grc_pp%topf(g) ! Check occurence of grid elevation is +ve while tpu elevation is -ve + topoElv = top_pp%elevation(t) + if (topoElv > max_tpuElv) then + max_tpuElv = topoElv + end if + if (topoElv < min_tpuElv) then + min_tpuElv = topoElv + end if + end do + elvrnge = max_tpuElv - min_tpuElv + + do t = grc_pp%topi(g), grc_pp%topf(g) + t2 = t - grc_pp%topi(g) + 1 + topoElv = top_pp%elevation(t) ! Topounit sfc elevation + + ! Downscale precipitation + if (mxElv == 0._r8 .or. precip_dwn == 1) then ! avoid dividing by 0 + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + else + if (precip_downscaling_method == 'FNM') then + call downscale_precip_to_topounit_FNM(mxElv,uovern_t,grdElv,topoElv,rain_g,snow_g,deltaR,deltaS,hrise) !Use FNM method + deltaRain(t2) = deltaR + deltaSnow(t2) = deltaS + sum_of_hrise = sum_of_hrise + hrise + else + call downscale_precip_to_topounit_ERMM(t,mxElv,grdElv,topoElv,rain_g, snow_g,elv_flag,elvrnge) ! Use ERMM method + end if + end if + + ! Downscale other fluxes + if (other_forcing_dwn == 1) then ! flag to turn on or off downscaling of other forcing + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + top_af%lwrad(t) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + ! Update top_as + top_as%tbot(t) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(t) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(t) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(t) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + else + call downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars, lnd2atm_vars, uaflag) + call downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars, lnd2atm_vars, uaflag) + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + sum_qbot_g = sum_qbot_g + top_pp%wtgcell(t)*top_as%qbot(t) + sum_wtsq_g = sum_wtsq_g + top_pp%wtgcell(t) + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + + ! Keep track of the gridcell-level weighted sum for later normalization. + ! + ! This gridcell-level weighted sum just includes points for which we do the + ! downscaling (e.g., glc_mec points). Thus the contributing weights + ! generally do not add to 1. So to do the normalization properly, we also + ! need to keep track of the weights that have contributed to this sum. + sum_lwrad_g = sum_lwrad_g + top_pp%wtgcell(t)*top_af%lwrad(t) + sum_wtslw_g = sum_wtslw_g + top_pp%wtgcell(t) + end if + end do + if (precip_downscaling_method == 'FNM') then + do t = grc_pp%topi(g), grc_pp%topf(g) + t2 = t - grc_pp%topi(g) + 1 + if (mxElv == 0.) then ! avoid dividing by 0 + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + else + top_af%rain(t) = rain_g + (deltaRain(t2) - (rain_g/mxElv)*(sum_of_hrise/numt_pg)) + top_af%snow(t) = snow_g + (deltaSnow(t2) - (snow_g/mxElv)*(sum_of_hrise/numt_pg)) + end if + + end do + deallocate(deltaRain) + deallocate(deltaSnow) + end if + + ! Precipitation partitioning using simple method following Jordan (1991) + do t = grc_pp%topi(g), grc_pp%topf(g) + crnt_temp_t = top_as%tbot(t) + if (crnt_temp_t > Ta_th3) then ! No snow or all rain + temp_r = top_af%rain(t) + top_af%snow(t) + temp_s = 0._r8 + else if (crnt_temp_t >= Ta_th2 .and. crnt_temp_t <= Ta_th3) then + temp_s = (top_af%snow(t) + top_af%rain(t))*0.6_r8 ! 0.6 fraction of the total precip is snow + temp_r = (top_af%snow(t) + top_af%rain(t)) - temp_s + else if (crnt_temp_t < Ta_th2 .and. crnt_temp_t > Ta_th1) then + tmp_Snow_frc = (crnt_temp_t-Ta_th2)*(0.4_r8/(Ta_th1-Ta_th2))+0.6_r8 ! Snow fraction value 1-0.6 = 0.4 snowfrc is 1 at t = Ta_th1 + temp_s = (top_af%snow(t) + top_af%rain(t))*tmp_Snow_frc ! 0.6 is snow fraction at t = Ta_th2 + temp_r = (top_af%snow(t) + top_af%rain(t)) - temp_s + else ! crnt_temp_t <= Ta_th1 ==> all snow + temp_s = top_af%snow(t) + top_af%rain(t) + temp_r = 0._r8 + + end if + + top_af%rain(t) = temp_r + top_af%snow(t) = temp_s + + end do + + else !grid has a single topounit + ! update top_af using grid level values + t = grc_pp%topi(g) + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + top_af%lwrad(t) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + + ! Update top_as + top_as%tbot(t) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(t) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(t) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(t) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + if (atm_gustiness) then + top_as%windbot(t) = sqrt(top_as%windbot(t)**2 + top_as%ugust(t)**2) + end if + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + + end if + + if (numt_pg > 1) then + + ! Normalize forc_qbot to conserve energy + + call build_normalization(orig_field=atm2lnd_vars%forc_q_not_downscaled_grc(g), & + sum_field=sum_qbot_g, sum_wts=sum_wtsq_g, norms=qbot_norm_g) + + do t = grc_pp%topi(g), grc_pp%topf(g) + top_as%qbot(t) = top_as%qbot(t) * qbot_norm_g + + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + end do + + + ! Normalize forc_lwrad_c(c) to conserve energy + + call build_normalization(orig_field=atm2lnd_vars%forc_lwrad_not_downscaled_grc(g), & + sum_field=sum_lwrad_g, sum_wts=sum_wtslw_g, norms=lwrad_norm_g) + + do t = grc_pp%topi(g), grc_pp%topf(g) + top_af%lwrad(t) = top_af%lwrad(t) * lwrad_norm_g + end do + + end if + + end subroutine downscale_atm_forcing_to_topounit_cpl_bypass + + !----------------------------------------------------------------------- + ! Downscale other atmospheric state variables + !----------------------------------------------------------------------- + subroutine downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + ! + ! !DESCRIPTION: + ! Downscale atmospheric forcing fields from gridcell to topounit + ! + ! Downscaling is done over topounits. + ! + ! !USES: + use elm_time_manager, only : get_nstep + use elm_varcon , only : rair, cpair, grav, lapse_glcmec + use elm_varcon , only : glcmec_rain_snow_threshold + use landunit_varcon , only : istice_mec + use elm_varctl , only : glcmec_downscale_rain_snow_convert + use domainMod , only : ldomain + use QsatMod , only : Qsat + ! + ! !ARGUMENTS: + integer , intent(in) :: g + integer , intent(in) :: t + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + ! + ! !LOCAL VARIABLES: + integer :: l, c, fc ! indices + integer :: clo, cc + integer :: nstep + + ! temporaries for topo downscaling + real(r8) :: hsurf_g,hsurf_t,Hbot + real(r8) :: zbot_g, tbot_g, pbot_g, thbot_g, qbot_g, qs_g, es_g + real(r8) :: zbot_t, tbot_t, pbot_t, thbot_t, qbot_t, qs_t, es_t + real(r8) :: egcm_t, rhos_t + real(r8) :: dum1, dum2 + + character(len=*), parameter :: subname = 'downscale_atm_state_to_topounit' + !----------------------------------------------------------------------- + + nstep = get_nstep() + + ! Downscale forc_t, forc_th, forc_q, forc_pbot, and forc_rho to columns. + ! For glacier_mec columns the downscaling is based on surface elevation. + ! For other columns the downscaling is a simple copy (above). + + ! This is a simple downscaling procedure + ! Note that forc_hgt, forc_u, and forc_v are not downscaled. + + hsurf_g = grc_pp%elevation(g) ! gridcell sfc elevation + hsurf_t = top_pp%elevation(t) ! topounit sfc elevation + tbot_g = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! atm sfc temp + thbot_g = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! atm sfc pot temp + qbot_g = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! atm sfc spec humid + pbot_g = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! atm sfc pressure + zbot_g = atm2lnd_vars%forc_hgt_grc(g) ! atm ref height + + zbot_t = zbot_g + tbot_t = tbot_g-lapse_glcmec*(hsurf_t-hsurf_g) ! sfc temp for column + + Hbot = rair*0.5_r8*(tbot_g+tbot_t)/grav ! scale ht at avg temp + pbot_t = pbot_g*exp(-(hsurf_t-hsurf_g)/Hbot) ! column sfc press + + ! Derivation of potential temperature calculation: + ! + ! The textbook definition would be: + ! thbot_c = tbot_c * (p0/pbot_c)^(rair/cpair) + ! + ! Note that pressure is related to scale height as: + ! pbot_c = p0 * exp(-zbot_c/H) + ! + ! Using Hbot in place of H, we get: + ! pbot_c = p0 * exp(-zbot_c/Hbot) + ! + ! Plugging this in to the textbook definition, then manipulating, we get: + ! thbot_c = tbot_c * (p0/(p0*exp(-zbot_c/Hbot)))^(rair/cpair) + ! = tbot_c * (1/exp(-zbot_c/Hbot))^(rair/cpair) + ! = tbot_c * (exp(zbot_c/Hbot))^(rair/cpair) + ! = tbot_c * exp((zbot_c/Hbot) * (rair/cpair)) + + thbot_t= tbot_t*exp((zbot_t/Hbot)*(rair/cpair)) ! pot temp calc + + call Qsat(tbot_g,pbot_g,es_g,dum1,qs_g,dum2) + call Qsat(tbot_t,pbot_t,es_t,dum1,qs_t,dum2) + + qbot_t = qbot_g*(qs_t/qs_g) + egcm_t = qbot_t*pbot_t/(0.622_r8+0.378_r8*qbot_t) + rhos_t = (pbot_t-0.378_r8*egcm_t) / (rair*tbot_t) + + top_as%tbot(t) = tbot_t + top_as%thbot(t) = thbot_t + top_as%qbot(t) = qbot_t + top_as%pbot(t) = pbot_t + +! call check_downscale_consistency(bounds, atm2lnd_vars) + + end subroutine downscale_atm_state_to_topounit_cpl_bypass + + !------------------------------------------------------- + ! Downscale longwave radiation place holder + subroutine downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + + ! !DESCRIPTION: + ! Downscale longwave radiation from gridcell to column + ! Must be done AFTER temperature downscaling + + ! !USES: + use elm_time_manager, only : get_nstep + use domainMod , only : ldomain + use landunit_varcon , only : istice_mec + use elm_varcon , only : lapse_glcmec + use elm_varctl , only : glcmec_downscale_longwave + + ! !ARGUMENTS: + integer , intent(in) :: g + integer , intent(in) :: t + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + + ! !LOCAL VARIABLES: + integer :: c,l,fc ! indices + integer :: nstep + real(r8) :: hsurf_t ! column-level elevation (m) + real(r8) :: hsurf_g ! gridcell-level elevation (m) + + real(r8) :: tair_g ! original gridcell mean air temperature + real(r8) :: tair_t ! downscaled topounit air temperature + real(r8) :: tsfc_g ! original gridcell surface temperature + real(r8) :: tsfc_t ! downscaled topounit surface temperature + real(r8) :: lwrad_g ! original gridcell mean LW radiation + real(r8) :: lwrad_t ! downscaled topounit LW radiation + real(r8) :: newsum_lwrad_g ! weighted sum of column-level lwrad after normalization + + character(len=*), parameter :: subname = 'downscale_longwave_to_topounit' + !----------------------------------------------------------------------- + + nstep = get_nstep() + + ! Do the downscaling + hsurf_g = grc_pp%elevation(g) + hsurf_t = top_pp%elevation(t) + + ! Here we assume that deltaLW = (dLW/dT)*(dT/dz)*deltaz + ! We get dLW/dT = 4*eps*sigma*T^3 = 4*LW/T from the Stefan-Boltzmann law, + ! evaluated at the mean temp. + ! We assume the same temperature lapse rate as above. + + tair_g = x2l(index_x2l_Sa_tbot,i) + tair_t = top_as%tbot(t) + lwrad_g = x2l(index_x2l_Faxa_lwdn,i) + top_af%lwrad(t) = lwrad_g - & + 4.0_r8 * lwrad_g/(0.5_r8*(tair_t+tair_g)) * & + lapse_glcmec * (hsurf_t - hsurf_g) + + end subroutine downscale_longwave_to_topounit_cpl_bypass + end module lnd_downscale_atm_forcing diff --git a/components/elm/src/cpl/lnd_import_export.F90 b/components/elm/src/cpl/lnd_import_export.F90 index ad81846196d..27c01e66818 100644 --- a/components/elm/src/cpl/lnd_import_export.F90 +++ b/components/elm/src/cpl/lnd_import_export.F90 @@ -1030,52 +1030,68 @@ subroutine lnd_import( bounds, x2l, atm2lnd_vars, glc2lnd_vars, lnd2atm_vars) end if end if - !set the topounit-level atmospheric state and flux forcings (bypass mode) + ! Adding topographic downscaling capability within CPL_BYPASS code block + ! PET, 7/12/2024 + if (use_atm_downscaling_to_topunit) then + atm2lnd_vars%forc_uovern = x2l(index_x2l_Sa_uovern,i) + atm2lnd_vars%forc_rain_not_downscaled_grc = forc_rainc + forc_rainl + atm2lnd_vars%forc_snow_not_downscaled_grc = forc_snowc + forc_snowl + + if(atm_gustiness) then + call endrun("Error: atm_gustiness not yet supported with multiple topounits (in CPL_BYPASS)") + end if + do topo = grc_pp%topi(g) , grc_pp%topf(g) + top_as%ugust(topo) = 0._r8 + end do + + call downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm_vars) + else + do topo = grc_pp%topi(g), grc_pp%topf(g) + top_as%tbot(topo) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(topo) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(topo) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(topo) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(topo) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(topo) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(topo) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + top_as%windbot(topo) = sqrt(top_as%ubot(topo)**2 + top_as%vbot(topo)**2) + ! Relative humidity (percent) + if (top_as%tbot(topo) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(topo))) + else + e = esati(tdc(top_as%tbot(topo))) + end if + qsat = 0.622_r8*e / (top_as%pbot(topo) - 0.378_r8*e) + top_as%rhbot(topo) = 100.0_r8*(top_as%qbot(topo) / qsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(topo) = o2_molar_const * top_as%pbot(topo) + ! air density (kg/m**3) - uses a temporary calculation of water vapor pressure (Pa) + vp = top_as%qbot(topo) * top_as%pbot(topo) / (0.622_r8 + 0.378_r8 * top_as%qbot(topo)) + top_as%rhobot(topo) = (top_as%pbot(topo) - 0.378_r8 * vp) / (rair * top_as%tbot(topo)) + top_af%rain(topo) = forc_rainc + forc_rainl ! sum of convective and large-scale rain + top_af%snow(topo) = forc_snowc + forc_snowl ! sum of convective and large-scale snow + top_af%solad(topo,2) = atm2lnd_vars%forc_solad_grc(g,2) ! forc_sollxy Atm flux W/m^2 + top_af%solad(topo,1) = atm2lnd_vars%forc_solad_grc(g,1) ! forc_solsxy Atm flux W/m^2 + top_af%solai(topo,2) = atm2lnd_vars%forc_solai_grc(g,2) ! forc_solldxy Atm flux W/m^2 + top_af%solai(topo,1) = atm2lnd_vars%forc_solai_grc(g,1) ! forc_solsdxy Atm flux W/m^2 + top_af%lwrad(topo) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) ! flwdsxy Atm flux W/m^2 + ! derived flux forcings + top_af%solar(topo) = top_af%solad(topo,2) + top_af%solad(topo,1) + & + top_af%solai(topo,2) + top_af%solai(topo,1) + end do + end if + + !set the topounit-level atmospheric variables that are not handled in downscaling code do topo = grc_pp%topi(g), grc_pp%topf(g) ! first, all the state forcings - top_as%tbot(topo) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K - top_as%thbot(topo) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K - top_as%pbot(topo) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa - top_as%qbot(topo) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg - top_as%ubot(topo) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s - top_as%vbot(topo) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s - if (implicit_stress) then + if (implicit_stress) then top_as%wsresp(topo) = 0._r8 ! Atm state m/s/Pa top_as%tau_est(topo) = 0._r8 ! Atm state Pa end if top_as%ugust(topo) = 0._r8 ! Atm state m/s - top_as%zbot(topo) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m - ! assign the state forcing fields derived from other inputs - ! Horizontal windspeed (m/s) - top_as%windbot(topo) = sqrt(top_as%ubot(topo)**2 + top_as%vbot(topo)**2) if (atm_gustiness) then top_as%windbot(topo) = sqrt(top_as%windbot(topo)**2 + top_as%ugust(topo)**2) end if - ! Relative humidity (percent) - if (top_as%tbot(topo) > SHR_CONST_TKFRZ) then - e = esatw(tdc(top_as%tbot(topo))) - else - e = esati(tdc(top_as%tbot(topo))) - end if - qsat = 0.622_r8*e / (top_as%pbot(topo) - 0.378_r8*e) - top_as%rhbot(topo) = 100.0_r8*(top_as%qbot(topo) / qsat) - ! partial pressure of oxygen (Pa) - top_as%po2bot(topo) = o2_molar_const * top_as%pbot(topo) - ! air density (kg/m**3) - uses a temporary calculation of water vapor pressure (Pa) - vp = top_as%qbot(topo) * top_as%pbot(topo) / (0.622_r8 + 0.378_r8 * top_as%qbot(topo)) - top_as%rhobot(topo) = (top_as%pbot(topo) - 0.378_r8 * vp) / (rair * top_as%tbot(topo)) - - ! second, all the flux forcings - top_af%rain(topo) = forc_rainc + forc_rainl ! sum of convective and large-scale rain - top_af%snow(topo) = forc_snowc + forc_snowl ! sum of convective and large-scale snow - top_af%solad(topo,2) = atm2lnd_vars%forc_solad_grc(g,2) ! forc_sollxy Atm flux W/m^2 - top_af%solad(topo,1) = atm2lnd_vars%forc_solad_grc(g,1) ! forc_solsxy Atm flux W/m^2 - top_af%solai(topo,2) = atm2lnd_vars%forc_solai_grc(g,2) ! forc_solldxy Atm flux W/m^2 - top_af%solai(topo,1) = atm2lnd_vars%forc_solai_grc(g,1) ! forc_solsdxy Atm flux W/m^2 - top_af%lwrad(topo) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) ! flwdsxy Atm flux W/m^2 - ! derived flux forcings - top_af%solar(topo) = top_af%solad(topo,2) + top_af%solad(topo,1) + & - top_af%solai(topo,2) + top_af%solai(topo,1) end do !----------------------------------------------------------------------------------------------------- diff --git a/components/elm/src/main/atm2lndType.F90 b/components/elm/src/main/atm2lndType.F90 index a00140ab2dd..82c96b971e0 100644 --- a/components/elm/src/main/atm2lndType.F90 +++ b/components/elm/src/main/atm2lndType.F90 @@ -146,6 +146,10 @@ module atm2lndType real(r8) , pointer :: t_mo_patch (:) => null() ! patch 30-day average temperature (Kelvin) real(r8) , pointer :: t_mo_min_patch (:) => null() ! patch annual min of t_mo (Kelvin) + ! Needed for FNM precip downscaling, when used within CPL_BYPASS + real(r8), pointer :: forc_uovern (:) => null() ! Froude number (dimensionless) + + contains procedure, public :: Init @@ -310,6 +314,7 @@ subroutine InitAllocate(this, bounds) allocate(this%forc_ndep_nitr_grc (begg:endg)) ; this%forc_ndep_nitr_grc (:) = ival allocate(this%forc_soilph_grc (begg:endg)) ; this%forc_soilph_grc (:) = ival end if + allocate(this%forc_uovern (begg:endg)) ; this%uovern (:) = ival end subroutine InitAllocate From 41884332cfdcb2781c2f0850e845feb116d994a6 Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Fri, 12 Jul 2024 18:08:59 -0400 Subject: [PATCH 045/366] Bug fixes for coupler bypass downscaling --- components/elm/src/cpl/lnd_downscale_atm_forcing.F90 | 11 +++++------ components/elm/src/main/atm2lndType.F90 | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 index 8c003bbf5e9..3b69158f49e 100644 --- a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 +++ b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 @@ -810,8 +810,7 @@ subroutine downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm ! ! !ARGUMENTS: integer , intent(in) :: g - real(r8) , intent(in) :: x2l(:,:) - type(atm2lnd_type) , intent(in) :: atm2atm_vars + type(atm2lnd_type) , intent(in) :: atm2lnd_vars type(lnd2atm_type) , intent(in) :: lnd2atm_vars ! @@ -995,8 +994,8 @@ subroutine downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & top_af%solai(t,2) + top_af%solai(t,1) else - call downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars, lnd2atm_vars, uaflag) - call downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars, lnd2atm_vars, uaflag) + call downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + call downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m @@ -1304,9 +1303,9 @@ subroutine downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) ! evaluated at the mean temp. ! We assume the same temperature lapse rate as above. - tair_g = x2l(index_x2l_Sa_tbot,i) + tair_g = atm2lnd_vars%forc_t_not_downscaled_grc(g) tair_t = top_as%tbot(t) - lwrad_g = x2l(index_x2l_Faxa_lwdn,i) + lwrad_g = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) top_af%lwrad(t) = lwrad_g - & 4.0_r8 * lwrad_g/(0.5_r8*(tair_t+tair_g)) * & lapse_glcmec * (hsurf_t - hsurf_g) diff --git a/components/elm/src/main/atm2lndType.F90 b/components/elm/src/main/atm2lndType.F90 index 82c96b971e0..f2facc77622 100644 --- a/components/elm/src/main/atm2lndType.F90 +++ b/components/elm/src/main/atm2lndType.F90 @@ -314,7 +314,7 @@ subroutine InitAllocate(this, bounds) allocate(this%forc_ndep_nitr_grc (begg:endg)) ; this%forc_ndep_nitr_grc (:) = ival allocate(this%forc_soilph_grc (begg:endg)) ; this%forc_soilph_grc (:) = ival end if - allocate(this%forc_uovern (begg:endg)) ; this%uovern (:) = ival + allocate(this%forc_uovern (begg:endg)) ; this%forc_uovern (:) = ival end subroutine InitAllocate From db237930afb0b619464f7d8c0a967e9bdd5853ba Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Wed, 6 Mar 2024 13:37:03 -0700 Subject: [PATCH 046/366] initial commit of adding shrub bending parameter --- components/elm/src/biogeochem/VegStructUpdateMod.F90 | 4 +++- .../elm/src/data_types/VegetationPropertiesType.F90 | 2 ++ components/elm/src/main/pftvarcon.F90 | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 5943da355a0..a8352ba2672 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -88,6 +88,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & z0mr => veg_vp%z0mr , & ! Input: [real(r8) (:) ] ratio of momentum roughness length to canopy top height (-) displar => veg_vp%displar , & ! Input: [real(r8) (:) ] ratio of displacement height to canopy top height (-) dwood => veg_vp%dwood , & ! Input: [real(r8) (:) ] density of wood (gC/m^3) + bend_parm => veg_vp%bend_parm , & ! Input: [real(r8) (:) ] shrub bending parameter after Sturm et al. 2005 (-) snow_depth => col_ws%snow_depth , & ! Input: [real(r8) (:) ] snow height (m) @@ -249,7 +250,8 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! Wang and Zeng, 2007. if (woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) - fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) + ! fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) ! Wang and Zeng, 2007 + fb = 1._r8 - ol / bend_parm(p)*max(1.e-06_r8, htop(p)-hbot(p)) ! Liston and Heimstra, 2011 else fb = 1._r8 - max(min(snow_depth(c),0.2_r8),0._r8)/0.2_r8 ! 0.2m is assumed !depth of snow required for complete burial of grasses diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 12f9a7b22ff..f2de380578d 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -151,6 +151,8 @@ module VegetationPropertiesType real(r8), pointer :: needleleaf(:) => null() !needleleaf or broadleaf real(r8), pointer :: nfixer(:) => null() !cablity of nitrogen fixation from atm. N2 + !NGEE Arctic + real(r8), pointer :: bend_parm(:) => null() ! shrub bending parameter contains procedure, public :: Init => veg_vp_init diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index fc5bfd54e7d..ccdb5afb657 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -311,6 +311,8 @@ module pftvarcon real(r8), allocatable :: gcbc_q(:) !effectiveness of surface cover in reducing runoff-driven erosion real(r8), allocatable :: gcbr_p(:) !effectiveness of roots in reducing rainfall-driven erosion real(r8), allocatable :: gcbr_q(:) !effectiveness of roots in reducing runoff-driven erosion + ! NGEE Arctic shrub bending + real(r8), allocatable :: bend_parm(:) ! parameter that describes the "stiffness" of shrub branches (Sturm et al. 2005, Liston and Hiemstra 2011) ! new pft properties, together with woody, crop, percrop, evergreen, stress_decid, season_decid, defined above, ! are introduced to define vegetation properties. This will be well defineing a pft so that no indices needed for codes. @@ -648,6 +650,8 @@ subroutine pftconrd allocate( needleleaf (0:mxpft) ) allocate( nfixer (0:mxpft) ) + ! NGEE arctic + allocate( bend_parm (0:mxpft) ) ! Set specific vegetation type values @@ -1076,6 +1080,10 @@ subroutine pftconrd end do end if + ! new NGEE parameters: + call ncd_io('bend_parm', bend_parm, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) bend_parm(:) = 1._r8 ! set to 1 if not on file - by default: no change. + ! NOTE: the following 5 PFT flags/options are addtions to 'woody', 'stress_decid', 'season_decid', ! 'evergreen', and 'crop', 'percop'. For default ELM, will be hard-coded; while for user-defined From f08448244f59e4b61e77a89b1f8ffe8e692a4d95 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Sun, 31 Mar 2024 14:34:25 -0600 Subject: [PATCH 047/366] Update snow burial of vegetation parameterization Enables two major changes to the parameterization that calculates LAI burial by snow: a) pulls stocking density and taper parameters that influence vegetation height from hard-wired values in biogeochem/VegStructUpdateMod.F90 to allow to be specified on the surface file (per main/pftvarcon) and b) updates the LAI burial by snow rate from Wang and Zeng, 2007 (linear burial) to form proposed by Liston and Hiemstra, 2011 (linear or non-linear) burial. The Liston and Heimstra (2011) formulation introduces two new parameters: 1) bendresist, which specifies how resistant branches are to bending under snow loading (1 = no bending, lim->0 branches bend very rapidly) and 2) vegshape, which specifies how the shape of the vegetation crown affects how rapidly it is buried by snow (1 = paraboloid; 2 = hemispheroid). These new parameters can also be specified on surface file. --- .../elm/src/biogeochem/VegStructUpdateMod.F90 | 38 +- .../data_types/VegetationPropertiesType.F90 | 25 +- components/elm/src/main/pftvarcon.F90 | 372 +++++++++--------- 3 files changed, 229 insertions(+), 206 deletions(-) diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index a8352ba2672..7c56fc6b58c 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -40,6 +40,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & use pftvarcon , only : noveg, woody, iscft, crop use pftvarcon , only : ncorn, ncornirrig, ztopmx, laimx use pftvarcon , only : nmiscanthus, nmiscanthusirrig, nswitchgrass, nswitchgrassirrig + use pftvarcon , only : bendresist, vegshape, stocking, taper use elm_time_manager , only : get_rad_step_size use elm_varctl , only : spinup_state, spinup_mortality_factor ! @@ -58,8 +59,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! !LOCAL VARIABLES: integer :: p,c,g ! indices integer :: fp ! lake filter indices - real(r8) :: taper ! ratio of height:radius_breast_height (tree allometry) - real(r8) :: stocking ! #stems / ha (stocking density) real(r8) :: ol ! thickness of canopy layer covered by snow (m) real(r8) :: fb ! fraction of canopy layer covered by snow real(r8) :: tlai_old ! for use in Zeng tsai formula @@ -88,7 +87,9 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & z0mr => veg_vp%z0mr , & ! Input: [real(r8) (:) ] ratio of momentum roughness length to canopy top height (-) displar => veg_vp%displar , & ! Input: [real(r8) (:) ] ratio of displacement height to canopy top height (-) dwood => veg_vp%dwood , & ! Input: [real(r8) (:) ] density of wood (gC/m^3) - bend_parm => veg_vp%bend_parm , & ! Input: [real(r8) (:) ] shrub bending parameter after Sturm et al. 2005 (-) + bendresist => veg_vp%bendresist , & ! Input: [real(r8) (:) ] resistance to bending under snow loading Sturm et al. 2005 (-) [0,1] + vegshape => veg_vp%vegshape , & ! Input: [real(r8) (:) ] vegetation shape for calculating snow buried fraction only (-) (Liston and Heimstra, 2011) + stocking => veg_vp%stocking , & ! Input: [real(r8) (:) ] Stocking density [stems / hectare] snow_depth => col_ws%snow_depth , & ! Input: [real(r8) (:) ] snow height (m) @@ -114,10 +115,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & dt = real( get_rad_step_size(), r8 ) - ! constant allometric parameters - taper = 200._r8 - stocking = 1000._r8 - ! convert from stems/ha -> stems/m^2 stocking = stocking / 10000._r8 @@ -161,24 +158,16 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & if (woody(ivt(p)) >= 1.0_r8) then - ! trees and shrubs - - ! if shrubs have a squat taper - if (woody(ivt(p)) == 2.0_r8) then - taper = 10._r8 - ! otherwise have a tall taper - else - taper = 200._r8 - end if - - ! trees and shrubs for now have a very simple allometry, with hard-wired - ! stem taper (height:radius) and hard-wired stocking density (#individuals/area) + ! trees and shrubs currently have a very simple allometry, based on + ! stem taper (height:radius) and stocking density (#individuals/area) + ! taper and stocking density can be set as input variables now to + ! change from default values set in pftvarcon.F90 if (spinup_state >= 1) then - htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper * taper)/ & - (SHR_CONST_PI * stocking * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper(p) * taper(p))/ & + (SHR_CONST_PI * stocking(p) * dwood(ivt(p))))**(1._r8/3._r8) else - htop(p) = ((3._r8 * deadstemc(p) * taper * taper)/ & - (SHR_CONST_PI * stocking * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * taper(p) * taper(p))/ & + (SHR_CONST_PI * stocking(p) * dwood(ivt(p))))**(1._r8/3._r8) end if ! Peter Thornton, 5/3/2004 @@ -250,8 +239,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! Wang and Zeng, 2007. if (woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) - ! fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) ! Wang and Zeng, 2007 - fb = 1._r8 - ol / bend_parm(p)*max(1.e-06_r8, htop(p)-hbot(p)) ! Liston and Heimstra, 2011 + fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(p) * (htop(p)-hbot(p)))) ** vegshape(p) else fb = 1._r8 - max(min(snow_depth(c),0.2_r8),0._r8)/0.2_r8 ! 0.2m is assumed !depth of snow required for complete burial of grasses diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index f2de380578d..2f06ea90032 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -115,7 +115,7 @@ module VegetationPropertiesType real(r8), pointer :: lamda_ptase => null()! critical value that incur biochemical production real(r8), pointer :: i_vc(:) => null() ! intercept of photosynthesis vcmax ~ leaf n content regression model real(r8), pointer :: s_vc(:) => null() ! slope of photosynthesis vcmax ~ leaf n content regression model - real(r8), pointer :: nsc_rtime(:) => null() ! non-structural carbon residence time + real(r8), pointer :: nsc_rtime(:) => null() ! non-structural carbon residence time real(r8), pointer :: pinit_beta1(:) => null() ! shaping parameter for P initialization real(r8), pointer :: pinit_beta2(:) => null() ! shaping parameter for P initialization real(r8), pointer :: alpha_nfix(:) => null() ! fraction of fixed N goes directly to plant @@ -151,8 +151,11 @@ module VegetationPropertiesType real(r8), pointer :: needleleaf(:) => null() !needleleaf or broadleaf real(r8), pointer :: nfixer(:) => null() !cablity of nitrogen fixation from atm. N2 - !NGEE Arctic - real(r8), pointer :: bend_parm(:) => null() ! shrub bending parameter + ! NGEE Arctic snow-vegetation interactions + real(r8), pointer :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011) + real(r8), pointer :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) + real(r8), pointer :: stocking(:) ! stocking density for pft (stems / hectare) + real(r8), pointer :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) contains procedure, public :: Init => veg_vp_init @@ -190,8 +193,10 @@ subroutine veg_vp_init(this) use pftvarcon , only : fnr, act25, kcha, koha, cpha, vcmaxha, jmaxha, tpuha use pftvarcon , only : lmrha, vcmaxhd, jmaxhd, tpuhd, lmrse, qe, theta_cj use pftvarcon , only : bbbopt, mbbopt, nstor, br_xr, tc_stress, lmrhd - ! new properties for flexible PFT + ! new properties for flexible PFT (NGEE Arctic IM4) use pftvarcon , only : climatezone, nonvascular, graminoid, iscft,needleleaf, nfixer + ! snow/vegetation interactions (NGEE Arctic IM3) + use pftvarcon , only : bendresist, stocking, vegshape, taper ! class (vegetation_properties_type) :: this @@ -324,6 +329,11 @@ subroutine veg_vp_init(this) allocate( this%needleleaf(0:numpft)) ; this%needleleaf(:) =spval allocate( this%nfixer(0:numpft)) ; this%nfixer(:) =spval ! ----------------------------------------------------------------------------------------------------------- + ! NGEE Arctic show-vegetation interactions + allocate(this%bendresist(0:numpft)) ; this%bendresist(:) =spval + allocate(this%vegshape(0:numpft)) ; this%vegshape(:) =spval + allocate(this%stocking(0:numpft)) ; this%stocking(:) =spval + allocate(this%taper(0:numpft)) ; this%taper(:) =spval do m = 0,numpft @@ -474,6 +484,13 @@ subroutine veg_vp_init(this) this%lamda_ptase = lamda_ptase this%tc_stress = tc_stress + ! NGEE Arctic - snow/vegetation interactions + do m = 0, numpft ! RPF - move up to earlier pft loops? + this%bendresist(m) = bendresist(m) + this%vegshape(m) = vegshape(m) + this%stocking(m) = stocking(m) + this%taper(m) = taper(m) + end do end subroutine veg_vp_init end module VegetationPropertiesType diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index ccdb5afb657..e06b5aba195 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -29,7 +29,7 @@ module pftvarcon ! ! Vegetation type constants ! - integer :: noveg !value for not vegetated + integer :: noveg !value for not vegetated integer :: ndllf_evr_tmp_tree !value for Needleleaf evergreen temperate tree integer :: ndllf_evr_brl_tree !value for Needleleaf evergreen boreal tree integer :: ndllf_dcd_brl_tree !value for Needleleaf deciduous boreal tree @@ -76,7 +76,7 @@ module pftvarcon integer :: nrtubers !value for root tubers, rain fed (rf) integer :: nrtubersirrig !value for root tubers, irrigated (ir) integer :: nsugarcane !value for sugarcane, rain fed (rf) - integer :: nsugarcaneirrig !value for sugarcane, irrigated (ir) + integer :: nsugarcaneirrig !value for sugarcane, irrigated (ir) integer :: nmiscanthus !value for miscanthus, rain fed (rf) integer :: nmiscanthusirrig !value for miscanthus, irrigated (ir) integer :: nswitchgrass !value for switchgrass, rain fed (rf) @@ -84,8 +84,8 @@ module pftvarcon integer :: npoplar !value for poplar, rain fed (rf) integer :: npoplarirrig !value for poplar, irrigated (ir) integer :: nwillow !value for willow, rain fed (rf) - integer :: nwillowirrig !value for willow, irrigated (ir) - + integer :: nwillowirrig !value for willow, irrigated (ir) + ! Number of crop functional types actually used in the model. This includes each CFT for ! which is_pft_known_to_model is true. Note that this includes irrigated crops even if ! irrigation is turned off in this run: it just excludes crop types that aren't handled @@ -171,7 +171,7 @@ module pftvarcon real(r8), allocatable :: minplanttemp(:) !mininum planting temperature used in Phenology (K) real(r8), allocatable :: senestemp(:) !senescence temperature for perennial crops used in Phenology (K) real(r8), allocatable :: min_days_senes(:) !minimum leaf age to allow for leaf senescence - real(r8), allocatable :: froot_leaf(:) !allocation parameter: new fine root C per new leaf C (gC/gC) + real(r8), allocatable :: froot_leaf(:) !allocation parameter: new fine root C per new leaf C (gC/gC) real(r8), allocatable :: stem_leaf(:) !allocation parameter: new stem c per new leaf C (gC/gC) real(r8), allocatable :: croot_stem(:) !allocation parameter: new coarse root C per new stem C (gC/gC) real(r8), allocatable :: flivewd(:) !allocation parameter: fraction of new wood that is live (phloem and ray parenchyma) (no units) @@ -215,7 +215,7 @@ module pftvarcon real(r8), allocatable :: fyield(:) !fraction of grain that is actually harvested real(r8), allocatable :: root_dmx(:) !maximum root depth - integer, parameter :: pftname_len = 40 ! max length of pftname + integer, parameter :: pftname_len = 40 ! max length of pftname character(len=:), allocatable :: pftname(:) !PFT description real(r8), parameter :: reinickerp = 1.6_r8 !parameter in allometric equation @@ -276,7 +276,7 @@ module pftvarcon real(r8), allocatable :: deadwdcp_obs_flex(:,:) !upper and lower range of dead wood (xylem and heartwood) C:P (gC/gP) ! Photosynthesis parameters real(r8), allocatable :: fnr(:) !fraction of nitrogen in RuBisCO - real(r8), allocatable :: act25(:) + real(r8), allocatable :: act25(:) real(r8), allocatable :: kcha(:) !Activation energy for kc real(r8), allocatable :: koha(:) !Activation energy for ko real(r8), allocatable :: cpha(:) !Activation energy for cp @@ -293,7 +293,7 @@ module pftvarcon real(r8), allocatable :: theta_cj(:) ! real(r8), allocatable :: bbbopt(:) !Ball-Berry stomatal conductance intercept real(r8), allocatable :: mbbopt(:) !Ball-Berry stomatal conductance slope - real(r8), allocatable :: nstor(:) !Nitrogen storage pool timescale + real(r8), allocatable :: nstor(:) !Nitrogen storage pool timescale real(r8), allocatable :: br_xr(:) !Base rate for excess respiration real(r8) :: tc_stress !Critial temperature for moisture stress real(r8), allocatable :: vcmax_np1(:) !vcmax~np relationship coefficient @@ -311,8 +311,12 @@ module pftvarcon real(r8), allocatable :: gcbc_q(:) !effectiveness of surface cover in reducing runoff-driven erosion real(r8), allocatable :: gcbr_p(:) !effectiveness of roots in reducing rainfall-driven erosion real(r8), allocatable :: gcbr_q(:) !effectiveness of roots in reducing runoff-driven erosion - ! NGEE Arctic shrub bending - real(r8), allocatable :: bend_parm(:) ! parameter that describes the "stiffness" of shrub branches (Sturm et al. 2005, Liston and Hiemstra 2011) + + ! NGEE Arctic snow-vegetation interactions + real(r8), allocatable :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011) + real(r8), allocatable :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) + real(r8), allocatable :: stocking(:) ! stocking density for pft (stems / hectare) + real(r8), allocatable :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) ! new pft properties, together with woody, crop, percrop, evergreen, stress_decid, season_decid, defined above, ! are introduced to define vegetation properties. This will be well defineing a pft so that no indices needed for codes. @@ -381,7 +385,7 @@ subroutine pftconrd ! and finally crops, ending with soybean ! DO NOT CHANGE THE ORDER -- WITHOUT MODIFYING OTHER PARTS OF THE CODE WHERE THE ORDER MATTERS! ! - character(len=pftname_len) :: expected_pftnames(0:mxpft) + character(len=pftname_len) :: expected_pftnames(0:mxpft) !----------------------------------------------------------------------- expected_pftnames( 0) = 'not_vegetated ' @@ -467,82 +471,82 @@ subroutine pftconrd allocate( rootb_par (0:mxpft) ) allocate( crop (0:mxpft) ) allocate( percrop (0:mxpft) ) - allocate( irrigated (0:mxpft) ) - allocate( smpso (0:mxpft) ) - allocate( smpsc (0:mxpft) ) - allocate( fnitr (0:mxpft) ) - allocate( slatop (0:mxpft) ) - allocate( dsladlai (0:mxpft) ) + allocate( irrigated (0:mxpft) ) + allocate( smpso (0:mxpft) ) + allocate( smpsc (0:mxpft) ) + allocate( fnitr (0:mxpft) ) + allocate( slatop (0:mxpft) ) + allocate( dsladlai (0:mxpft) ) allocate( leafcn (0:mxpft) ) - allocate( flnr (0:mxpft) ) - allocate( woody (0:mxpft) ) - allocate( lflitcn (0:mxpft) ) - allocate( frootcn (0:mxpft) ) - allocate( livewdcn (0:mxpft) ) - allocate( deadwdcn (0:mxpft) ) - - ! add phosphorus - allocate( leafcp (0:mxpft) ) - allocate( lflitcp (0:mxpft) ) - allocate( frootcp (0:mxpft) ) - allocate( livewdcp (0:mxpft) ) - allocate( deadwdcp (0:mxpft) ) - - allocate( grperc (0:mxpft) ) - allocate( grpnow (0:mxpft) ) + allocate( flnr (0:mxpft) ) + allocate( woody (0:mxpft) ) + allocate( lflitcn (0:mxpft) ) + allocate( frootcn (0:mxpft) ) + allocate( livewdcn (0:mxpft) ) + allocate( deadwdcn (0:mxpft) ) + + ! add phosphorus + allocate( leafcp (0:mxpft) ) + allocate( lflitcp (0:mxpft) ) + allocate( frootcp (0:mxpft) ) + allocate( livewdcp (0:mxpft) ) + allocate( deadwdcp (0:mxpft) ) + + allocate( grperc (0:mxpft) ) + allocate( grpnow (0:mxpft) ) allocate( rootprof_beta (0:mxpft) ) allocate( mergetoelmpft (0:mxpft) ) allocate( is_pft_known_to_model (0:mxpft) ) - allocate( graincn (0:mxpft) ) - allocate( graincp (0:mxpft) ) - allocate( mxtmp (0:mxpft) ) - allocate( baset (0:mxpft) ) - allocate( declfact (0:mxpft) ) - allocate( bfact (0:mxpft) ) - allocate( aleaff (0:mxpft) ) - allocate( arootf (0:mxpft) ) - allocate( astemf (0:mxpft) ) - allocate( arooti (0:mxpft) ) - allocate( fleafi (0:mxpft) ) - allocate( allconsl (0:mxpft) ) - allocate( allconss (0:mxpft) ) - allocate( ztopmx (0:mxpft) ) - allocate( laimx (0:mxpft) ) - allocate( gddmin (0:mxpft) ) - allocate( hybgdd (0:mxpft) ) - allocate( lfemerg (0:mxpft) ) - allocate( grnfill (0:mxpft) ) - allocate( mxmat (0:mxpft) ) + allocate( graincn (0:mxpft) ) + allocate( graincp (0:mxpft) ) + allocate( mxtmp (0:mxpft) ) + allocate( baset (0:mxpft) ) + allocate( declfact (0:mxpft) ) + allocate( bfact (0:mxpft) ) + allocate( aleaff (0:mxpft) ) + allocate( arootf (0:mxpft) ) + allocate( astemf (0:mxpft) ) + allocate( arooti (0:mxpft) ) + allocate( fleafi (0:mxpft) ) + allocate( allconsl (0:mxpft) ) + allocate( allconss (0:mxpft) ) + allocate( ztopmx (0:mxpft) ) + allocate( laimx (0:mxpft) ) + allocate( gddmin (0:mxpft) ) + allocate( hybgdd (0:mxpft) ) + allocate( lfemerg (0:mxpft) ) + allocate( grnfill (0:mxpft) ) + allocate( mxmat (0:mxpft) ) allocate( mnNHplantdate (0:mxpft) ) allocate( mxNHplantdate (0:mxpft) ) allocate( mnSHplantdate (0:mxpft) ) allocate( mxSHplantdate (0:mxpft) ) - allocate( planttemp (0:mxpft) ) - allocate( minplanttemp (0:mxpft) ) + allocate( planttemp (0:mxpft) ) + allocate( minplanttemp (0:mxpft) ) allocate( senestemp (0:mxpft) ) allocate( min_days_senes (0:mxpft) ) - allocate( froot_leaf (0:mxpft) ) - allocate( stem_leaf (0:mxpft) ) - allocate( croot_stem (0:mxpft) ) - allocate( flivewd (0:mxpft) ) - allocate( fcur (0:mxpft) ) - allocate( lf_flab (0:mxpft) ) - allocate( lf_fcel (0:mxpft) ) - allocate( lf_flig (0:mxpft) ) - allocate( fr_flab (0:mxpft) ) - allocate( fr_fcel (0:mxpft) ) - allocate( fr_flig (0:mxpft) ) - allocate( leaf_long (0:mxpft) ) + allocate( froot_leaf (0:mxpft) ) + allocate( stem_leaf (0:mxpft) ) + allocate( croot_stem (0:mxpft) ) + allocate( flivewd (0:mxpft) ) + allocate( fcur (0:mxpft) ) + allocate( lf_flab (0:mxpft) ) + allocate( lf_fcel (0:mxpft) ) + allocate( lf_flig (0:mxpft) ) + allocate( fr_flab (0:mxpft) ) + allocate( fr_fcel (0:mxpft) ) + allocate( fr_flig (0:mxpft) ) + allocate( leaf_long (0:mxpft) ) allocate( froot_long (0:mxpft) ) - allocate( evergreen (0:mxpft) ) - allocate( stress_decid (0:mxpft) ) - allocate( season_decid (0:mxpft) ) - allocate( pconv (0:mxpft) ) - allocate( pprod10 (0:mxpft) ) - allocate( pprod100 (0:mxpft) ) - allocate( pprodharv10 (0:mxpft) ) + allocate( evergreen (0:mxpft) ) + allocate( stress_decid (0:mxpft) ) + allocate( season_decid (0:mxpft) ) + allocate( pconv (0:mxpft) ) + allocate( pprod10 (0:mxpft) ) + allocate( pprod100 (0:mxpft) ) + allocate( pprodharv10 (0:mxpft) ) allocate( cc_leaf (0:mxpft) ) allocate( cc_lstem (0:mxpft) ) allocate( cc_dstem (0:mxpft) ) @@ -557,12 +561,12 @@ subroutine pftconrd allocate( fsr_pft (0:mxpft) ) allocate( fd_pft (0:mxpft) ) allocate( manunitro (0:mxpft) ) - allocate( fleafcn (0:mxpft) ) - allocate( ffrootcn (0:mxpft) ) + allocate( fleafcn (0:mxpft) ) + allocate( ffrootcn (0:mxpft) ) allocate( fstemcn (0:mxpft) ) allocate( presharv (0:mxpft) ) allocate( convfact (0:mxpft) ) - allocate( fyield (0:mxpft) ) + allocate( fyield (0:mxpft) ) allocate( root_dmx (0:mxpft) ) if (use_crop) then @@ -575,14 +579,14 @@ subroutine pftconrd allocate( VMAX_PLANT_NO3(0:mxpft) ) allocate( VMAX_PLANT_P(0:mxpft) ) allocate( VMAX_MINSURF_P_vr(1:nlevdecomp_full,0:nsoilorder)) - allocate( KM_PLANT_NH4(0:mxpft) ) + allocate( KM_PLANT_NH4(0:mxpft) ) allocate( KM_PLANT_NO3(0:mxpft) ) allocate( KM_PLANT_P(0:mxpft) ) allocate( KM_MINSURF_P_vr(1:nlevdecomp_full,0:nsoilorder)) allocate( decompmicc_patch_vr (1:nlevdecomp_full,0:mxpft)) allocate( VMAX_PTASE(0:mxpft)) - allocate( i_vc (0:mxpft) ) - allocate( s_vc (0:mxpft) ) + allocate( i_vc (0:mxpft) ) + allocate( s_vc (0:mxpft) ) allocate( nsc_rtime (0:mxpft) ) allocate( pinit_beta1 (0:nsoilorder)) allocate( pinit_beta2 (0:nsoilorder)) @@ -595,22 +599,22 @@ subroutine pftconrd allocate( VMAX_NFIX (0:mxpft) ) allocate( KM_NFIX (0:mxpft) ) ! new stoichiometry - allocate( leafcn_obs (0:mxpft) ) - allocate( frootcn_obs (0:mxpft) ) + allocate( leafcn_obs (0:mxpft) ) + allocate( frootcn_obs (0:mxpft) ) allocate( livewdcn_obs (0:mxpft) ) - allocate( deadwdcn_obs (0:mxpft) ) - allocate( leafcp_obs (0:mxpft) ) - allocate( frootcp_obs (0:mxpft) ) + allocate( deadwdcn_obs (0:mxpft) ) + allocate( leafcp_obs (0:mxpft) ) + allocate( frootcp_obs (0:mxpft) ) allocate( livewdcp_obs (0:mxpft) ) - allocate( deadwdcp_obs (0:mxpft) ) - allocate( leafcn_obs_flex (0:mxpft,1:2) ) - allocate( frootcn_obs_flex (0:mxpft,1:2) ) + allocate( deadwdcp_obs (0:mxpft) ) + allocate( leafcn_obs_flex (0:mxpft,1:2) ) + allocate( frootcn_obs_flex (0:mxpft,1:2) ) allocate( livewdcn_obs_flex (0:mxpft,1:2) ) - allocate( deadwdcn_obs_flex (0:mxpft,1:2) ) - allocate( leafcp_obs_flex (0:mxpft,1:2) ) - allocate( frootcp_obs_flex (0:mxpft,1:2) ) + allocate( deadwdcn_obs_flex (0:mxpft,1:2) ) + allocate( leafcp_obs_flex (0:mxpft,1:2) ) + allocate( frootcp_obs_flex (0:mxpft,1:2) ) allocate( livewdcp_obs_flex (0:mxpft,1:2) ) - allocate( deadwdcp_obs_flex (0:mxpft,1:2) ) + allocate( deadwdcp_obs_flex (0:mxpft,1:2) ) allocate( vcmax_np1 (0:mxpft) ) allocate( vcmax_np2 (0:mxpft) ) allocate( vcmax_np3 (0:mxpft) ) @@ -650,12 +654,16 @@ subroutine pftconrd allocate( needleleaf (0:mxpft) ) allocate( nfixer (0:mxpft) ) - ! NGEE arctic - allocate( bend_parm (0:mxpft) ) + ! NGEE arctic snow-vegetation interactions + allocate( bendresist (0:mxpft) ) + allocate( vegshape (0:mxpft) ) + allocate( stocking (0:mxpft) ) + allocate( taper (0:mxpft) ) + ! Set specific vegetation type values - call ncd_io('pftname',pftname, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('pftname',pftname, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) ! checking if using standard pft-names and default properties @@ -756,16 +764,16 @@ subroutine pftconrd if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) call ncd_io('fr_flab',fr_flab, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fr_fcel',fr_fcel, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('fr_fcel',fr_fcel, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fr_flig',fr_flig, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('fr_flig',fr_flig, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('leaf_long',leaf_long, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('leaf_long',leaf_long, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) call ncd_io('froot_long',froot_long, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv) froot_long = leaf_long !if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('evergreen',evergreen, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('evergreen',evergreen, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) call ncd_io('stress_decid',stress_decid, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) @@ -805,95 +813,95 @@ subroutine pftconrd call ncd_io('rootprof_beta',rootprof_beta, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) end if - call ncd_io('pconv',pconv, 'read', ncid, readvar=readv) + call ncd_io('pconv',pconv, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('pprod10',pprod10, 'read', ncid, readvar=readv) + call ncd_io('pprod10',pprod10, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('pprodharv10',pprodharv10, 'read', ncid, readvar=readv) + call ncd_io('pprodharv10',pprodharv10, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('pprod100',pprod100, 'read', ncid, readvar=readv) + call ncd_io('pprod100',pprod100, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('graincn',graincn, 'read', ncid, readvar=readv) + call ncd_io('graincn',graincn, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('graincp',graincp, 'read', ncid, readvar=readv) + call ncd_io('graincp',graincp, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('mxtmp',mxtmp, 'read', ncid, readvar=readv) + call ncd_io('mxtmp',mxtmp, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('baset',baset, 'read', ncid, readvar=readv) + call ncd_io('baset',baset, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('declfact',declfact, 'read', ncid, readvar=readv) + call ncd_io('declfact',declfact, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('bfact',bfact, 'read', ncid, readvar=readv) + call ncd_io('bfact',bfact, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('aleaff',aleaff, 'read', ncid, readvar=readv) + call ncd_io('aleaff',aleaff, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('arootf',arootf, 'read', ncid, readvar=readv) + call ncd_io('arootf',arootf, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('astemf',astemf, 'read', ncid, readvar=readv) + call ncd_io('astemf',astemf, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('arooti',arooti, 'read', ncid, readvar=readv) + call ncd_io('arooti',arooti, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fleafi',fleafi, 'read', ncid, readvar=readv) + call ncd_io('fleafi',fleafi, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('allconsl',allconsl, 'read', ncid, readvar=readv) + call ncd_io('allconsl',allconsl, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('allconss',allconss, 'read', ncid, readvar=readv) + call ncd_io('allconss',allconss, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('crop',crop, 'read', ncid, readvar=readv) + call ncd_io('crop',crop, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('irrigated',irrigated, 'read', ncid, readvar=readv) + call ncd_io('irrigated',irrigated, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('ztopmx',ztopmx, 'read', ncid, readvar=readv) + call ncd_io('ztopmx',ztopmx, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('laimx',laimx, 'read', ncid, readvar=readv) + call ncd_io('laimx',laimx, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('gddmin',gddmin, 'read', ncid, readvar=readv) + call ncd_io('gddmin',gddmin, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('hybgdd',hybgdd, 'read', ncid, readvar=readv) + call ncd_io('hybgdd',hybgdd, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('lfemerg',lfemerg, 'read', ncid, readvar=readv) + call ncd_io('lfemerg',lfemerg, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('grnfill',grnfill, 'read', ncid, readvar=readv) + call ncd_io('grnfill',grnfill, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('mxmat',mxmat, 'read', ncid, readvar=readv) + call ncd_io('mxmat',mxmat, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('cc_leaf', cc_leaf, 'read', ncid, readvar=readv) + call ncd_io('cc_leaf', cc_leaf, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('cc_lstem',cc_lstem, 'read', ncid, readvar=readv) + call ncd_io('cc_lstem',cc_lstem, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('cc_dstem',cc_dstem, 'read', ncid, readvar=readv) + call ncd_io('cc_dstem',cc_dstem, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('cc_other',cc_other, 'read', ncid, readvar=readv) + call ncd_io('cc_other',cc_other, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_leaf', fm_leaf, 'read', ncid, readvar=readv) + call ncd_io('fm_leaf', fm_leaf, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_lstem',fm_lstem, 'read', ncid, readvar=readv) + call ncd_io('fm_lstem',fm_lstem, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_dstem',fm_dstem, 'read', ncid, readvar=readv) + call ncd_io('fm_dstem',fm_dstem, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_other',fm_other, 'read', ncid, readvar=readv) + call ncd_io('fm_other',fm_other, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_root', fm_root, 'read', ncid, readvar=readv) + call ncd_io('fm_root', fm_root, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_lroot',fm_lroot, 'read', ncid, readvar=readv) + call ncd_io('fm_lroot',fm_lroot, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_droot',fm_droot, 'read', ncid, readvar=readv) + call ncd_io('fm_droot',fm_droot, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fsr_pft', fsr_pft, 'read', ncid, readvar=readv) + call ncd_io('fsr_pft', fsr_pft, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fd_pft', fd_pft, 'read', ncid, readvar=readv) + call ncd_io('fd_pft', fd_pft, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('planting_temp',planttemp, 'read', ncid, readvar=readv) + call ncd_io('planting_temp',planttemp, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('min_planting_temp',minplanttemp, 'read', ncid, readvar=readv) + call ncd_io('min_planting_temp',minplanttemp, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('min_NH_planting_date',mnNHplantdate, 'read', ncid, readvar=readv) + call ncd_io('min_NH_planting_date',mnNHplantdate, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('min_SH_planting_date',mnSHplantdate, 'read', ncid, readvar=readv) + call ncd_io('min_SH_planting_date',mnSHplantdate, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('max_NH_planting_date',mxNHplantdate, 'read', ncid, readvar=readv) + call ncd_io('max_NH_planting_date',mxNHplantdate, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('max_SH_planting_date',mxSHplantdate, 'read', ncid, readvar=readv) + call ncd_io('max_SH_planting_date',mxSHplantdate, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) if (nu_com .ne. 'RD' ) then @@ -901,40 +909,40 @@ subroutine pftconrd ! These are soil parameters and used for both FATES and big leaf ELM call ncd_io('VMAX_MINSURF_P_vr',VMAX_MINSURF_P_vr, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in soil order VMAX_MINSURF_P_vr'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_MINSURF_P_vr',KM_MINSURF_P_vr, 'read', ncid, readvar=readv) + call ncd_io('KM_MINSURF_P_vr',KM_MINSURF_P_vr, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in soil order KM_MINSURF_P_vr'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_DECOMP_NH4',KM_DECOMP_NH4, 'read', ncid, readvar=readv) + call ncd_io('KM_DECOMP_NH4',KM_DECOMP_NH4, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_DECOMP_NH4'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_DECOMP_NO3',KM_DECOMP_NO3, 'read', ncid, readvar=readv) + call ncd_io('KM_DECOMP_NO3',KM_DECOMP_NO3, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_DECOMP_NO3'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_DECOMP_P',KM_DECOMP_P, 'read', ncid, readvar=readv) + call ncd_io('KM_DECOMP_P',KM_DECOMP_P, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_DECOMP_P'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_NIT',KM_NIT, 'read', ncid, readvar=readv) + call ncd_io('KM_NIT',KM_NIT, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_NIT'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_DEN',KM_DEN, 'read', ncid, readvar=readv) + call ncd_io('KM_DEN',KM_DEN, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_DEN'//errMsg(__FILE__, __LINE__)) call ncd_io('pinit_beta1',pinit_beta1, 'read', ncid, readvar=readv) if ( .not. readv ) pinit_beta1(:) = 0.5_r8 call ncd_io('pinit_beta2',pinit_beta2, 'read', ncid, readvar=readv) if ( .not. readv ) pinit_beta2(:) = 0.1_r8 - + if(.not.use_fates) then - - call ncd_io('VMAX_PLANT_NH4',VMAX_PLANT_NH4, 'read', ncid, readvar=readv) + + call ncd_io('VMAX_PLANT_NH4',VMAX_PLANT_NH4, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft VMAX_PLANT_NH4'//errMsg(__FILE__, __LINE__)) - call ncd_io('VMAX_PLANT_NO3',VMAX_PLANT_NO3, 'read', ncid, readvar=readv) + call ncd_io('VMAX_PLANT_NO3',VMAX_PLANT_NO3, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft VMAX_PLANT_NO3'//errMsg(__FILE__, __LINE__)) - call ncd_io('VMAX_PLANT_P',VMAX_PLANT_P, 'read', ncid, readvar=readv) + call ncd_io('VMAX_PLANT_P',VMAX_PLANT_P, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft VMAX_PLANT_P'//errMsg(__FILE__, __LINE__)) - - call ncd_io('KM_PLANT_NH4',KM_PLANT_NH4, 'read', ncid, readvar=readv) + + call ncd_io('KM_PLANT_NH4',KM_PLANT_NH4, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft KM_PLANT_NH4'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_PLANT_NO3',KM_PLANT_NO3, 'read', ncid, readvar=readv) + call ncd_io('KM_PLANT_NO3',KM_PLANT_NO3, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft KM_PLANT_NO3'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_PLANT_P',KM_PLANT_P, 'read', ncid, readvar=readv) + call ncd_io('KM_PLANT_P',KM_PLANT_P, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft KM_PLANT_P'//errMsg(__FILE__, __LINE__)) - - call ncd_io('decompmicc_patch_vr',decompmicc_patch_vr, 'read', ncid, readvar=readv) + + call ncd_io('decompmicc_patch_vr',decompmicc_patch_vr, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft decompmicc_patch_vr'//errMsg(__FILE__, __LINE__)) call ncd_io('alpha_nfix',alpha_nfix, 'read', ncid, readvar=readv) if ( .not. readv ) alpha_nfix(:)=0._r8 @@ -948,19 +956,19 @@ subroutine pftconrd if ( .not. readv ) ccost_ptase(:)=0._r8 call ncd_io('ncost_ptase',ncost_ptase, 'read', ncid, readvar=readv) if ( .not. readv ) ncost_ptase(:)=0._r8 - call ncd_io('VMAX_NFIX',VMAX_NFIX, 'read', ncid, readvar=readv) + call ncd_io('VMAX_NFIX',VMAX_NFIX, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in VMAX_NFIX'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_NFIX',KM_NFIX, 'read', ncid, readvar=readv) + call ncd_io('KM_NFIX',KM_NFIX, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_NFIX'//errMsg(__FILE__, __LINE__)) - call ncd_io('VMAX_PTASE',VMAX_PTASE, 'read', ncid, readvar=readv) + call ncd_io('VMAX_PTASE',VMAX_PTASE, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in VMAX_PTASE'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_PTASE',KM_PTASE, 'read', ncid, readvar=readv) + call ncd_io('KM_PTASE',KM_PTASE, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_PTASE'//errMsg(__FILE__, __LINE__)) - call ncd_io('lamda_ptase',lamda_ptase, 'read', ncid, readvar=readv) + call ncd_io('lamda_ptase',lamda_ptase, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in lamda_ptase'//errMsg(__FILE__, __LINE__)) - call ncd_io('i_vc',i_vc, 'read', ncid, readvar=readv) + call ncd_io('i_vc',i_vc, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in i_vc'//errMsg(__FILE__, __LINE__)) - call ncd_io('s_vc',s_vc, 'read', ncid, readvar=readv) + call ncd_io('s_vc',s_vc, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in s_vc'//errMsg(__FILE__, __LINE__)) call ncd_io('nsc_rtime',nsc_rtime, 'read', ncid, readvar=readv) if ( .not. readv ) nsc_rtime(:) = 1.0_r8 @@ -1022,13 +1030,13 @@ subroutine pftconrd call ncd_io('fnr', fnr, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('act25', act25, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) + if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('kcha', kcha, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) + if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('koha', koha, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) + if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('cpha', cpha, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) + if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('vcmaxha', vcmaxha, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('jmaxha', jmaxha, 'read', ncid, readvar=readv, posNOTonfile=.true.) @@ -1080,10 +1088,20 @@ subroutine pftconrd end do end if - ! new NGEE parameters: - call ncd_io('bend_parm', bend_parm, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if (.not. readv ) bend_parm(:) = 1._r8 ! set to 1 if not on file - by default: no change. - + ! NGEE-Arctic snow-vegetation parameters + call ncd_io('bendresist', bendresist, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) bendresist(:) = 1._r8 + call ncd_io('vegshape', vegshape, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) vegshape(:) = 1._r8 + call ncd_io('stocking', stocking, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) stocking(:) = 1000._r8 + call ncd_io('taper', taper, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) then + taper(:) = 200._r8 + do i = 0, npft - 1 + if (woody(i) == 2) taper(i) = 10.0_r8 ! shrubs + end do + end if ! NOTE: the following 5 PFT flags/options are addtions to 'woody', 'stress_decid', 'season_decid', ! 'evergreen', and 'crop', 'percop'. For default ELM, will be hard-coded; while for user-defined @@ -1162,7 +1180,7 @@ subroutine pftconrd do i = 0, mxpft if(.not. use_crop .and. i > mxpft_nc) EXIT ! exit the do loop - + ! (FATES-INTERF) Later, depending on how the team plans to structure the crop model ! or other modules that co-exist while FATES is on, we may want to preserve these pft definitions ! on non-fates columns. For now, they are incompatible, and this check is warranted (rgk 04-2017) From f9799c62ed603a7949659b9b2e68f11433b38cfd Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Fri, 29 Mar 2024 08:16:37 -0600 Subject: [PATCH 048/366] Update ci-compile-test.yml --- .github/workflows/ci-compile-test.yml | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/ci-compile-test.yml diff --git a/.github/workflows/ci-compile-test.yml b/.github/workflows/ci-compile-test.yml new file mode 100644 index 00000000000..90e592630c6 --- /dev/null +++ b/.github/workflows/ci-compile-test.yml @@ -0,0 +1,56 @@ +name: CI +on: + push: + branches: + - '**' +jobs: + container-test-job: + runs-on: ubuntu-latest + container: + image: rfiorella/e3sm-dev:latest + steps: + - name: Check out the NGEE-Arctic-E3SM repo + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + submodules: true + - name: Setup cime and input datafiles + run: | + mkdir ~/.cime + cp /home/e3smuser/.cime/* /github/home/.cime/ + git clone https://github.com/rfiorella/pt-e3sm-inputdata /home/e3smuser/inputdata + cd /home/e3smuser/inputdata/lnd/clm2/firedata + tar -zxvf clmforc.Li_2012_hdm_0.5x0.5_AVHRR_simyr1850-2010_c130401.nc.tar.gz + cd /home/e3smuser/inputdata/atm/datm7/NASA_LIS/ + tar -zxvf clmforc.Li_2012_climo1995-2011.T62.lnfm_c130327.nc.tar.gz + tar -zxvf clmforc.Li_2012_climo1995-2011.T62.lnfm_Total_c140423.nc.tar.gz + ls -v /home/e3smuser/inputdata/atm/datm7/NASA_LIS/ + - name: create new case and setup + run: | + cd cime/scripts && ./create_newcase --mach docker --res ELM_USRDAT --compset ICB1850CNPRDCTCBC --case ~/output/ci_test \ + && cd ~/output/ci_test + ./xmlchange MOSART_MODE=NULL,DOUT_S=FALSE,DIN_LOC_ROOT=/home/e3smuser/inputdata + ./xmlchange DIN_LOC_ROOT_CLMFORC=/home/e3smuser/inputdata/atm/datm7 + ./xmlchange ELM_USRDAT_NAME=1x1pt_AK-TLG + ./xmlchange ATM_DOMAIN_PATH=/home/e3smuser/inputdata/share/domains/domain.clm + ./xmlchange LND_DOMAIN_PATH=/home/e3smuser/inputdata/share/domains/domain.clm + ./xmlchange ATM_DOMAIN_FILE=domain.lnd.1x1pt_teller-GRID_navy.nc + ./xmlchange LND_DOMAIN_FILE=domain.lnd.1x1pt_teller-GRID_navy.nc + ./xmlchange PIO_TYPENAME=netcdf,NTASKS=1,NTHRDS=1 + ./xmlchange STOP_OPTION=nyears,STOP_N=5 + ./xmlchange BATCH_SYSTEM=none + echo "&clm_inparm" >> user_nl_elm + echo " do_budgets = .false." >> user_nl_elm + echo " use_nofire = .true." >> user_nl_elm + echo " metdata_type = 'gswp3'" >> user_nl_elm + echo " metdata_bypass = '/home/e3smuser/inputdata/atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v2.c180716_NGEE-Grid/cpl_bypass_teller-Grid'" >> user_nl_elm + echo " fsurdat = '/home/e3smuser/inputdata/lnd/clm2/surfdata_map/surfdata_1x1pt_teller-GRID_simyr1850_c360x720_c171002.nc'" >> user_nl_elm + echo " stream_fldfilename_popdens = '/home/e3smuser/inputdata/lnd/clm2/firedata/clmforc.Li_2012_hdm_0.5x0.5_AVHRR_simyr1850-2010_c130401.nc'" >> user_nl_elm + ./case.setup + # add coupler bypass flag to cmake macros. + echo 'string(APPEND CPPDEFS " -DCPL_BYPASS")' >> cmake_macros/universal.cmake + ./case.build || cat /home/e3smuser/output/ci_test/bld/e3sm.bldlog.* + ./case.submit --no-batch + cat /home/e3smuser/output/ci_test/run/e3sm.log.* + cat /home/e3smuser/output/ci_test/run/lnd.log.* + From 7a2dd9c7c0ee073197f86a3ba72e8400e0ec492d Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Tue, 2 Apr 2024 07:17:15 -0600 Subject: [PATCH 049/366] hard-code pft numbers in pftvarcon for default taper values Temporary change, pending merge of IM4 updates. IM4 removed a lot of the hard-coded tests that looked for particular pfts by their assigned number. this required modifying the helper function "woody" to be a non-binary (i.e., now 0 = herbaceous, 1 = woody tree, 2 = woody shrub), but this updated function isn't available to IM3 currently. --- components/elm/src/main/pftvarcon.F90 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index e06b5aba195..e433d1f305a 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -1099,7 +1099,9 @@ subroutine pftconrd if (.not. readv ) then taper(:) = 200._r8 do i = 0, npft - 1 - if (woody(i) == 2) taper(i) = 10.0_r8 ! shrubs + ! RPF 240331 - need to revisit this section when integrating with IM4, + ! to make consistent with attempt to remove hard-coded pft numbers. + if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) taper(i) = 10.0_r8 ! shrubs end do end if From 702f73c691f521fffebd84b93b661b9e33e9a944 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Fri, 14 Jun 2024 17:57:58 -0600 Subject: [PATCH 050/366] add bounds checking on bendresist and vegshape parameters --- .../elm/src/biogeochem/VegStructUpdateMod.F90 | 13 +++++-------- components/elm/src/main/pftvarcon.F90 | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 7c56fc6b58c..868c417ff5f 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -115,9 +115,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & dt = real( get_rad_step_size(), r8 ) - ! convert from stems/ha -> stems/m^2 - stocking = stocking / 10000._r8 - ! patch loop do fp = 1,num_soilp p = filter_soilp(fp) @@ -163,11 +160,11 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! taper and stocking density can be set as input variables now to ! change from default values set in pftvarcon.F90 if (spinup_state >= 1) then - htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper(p) * taper(p))/ & - (SHR_CONST_PI * stocking(p) * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper(ivt(p)) * taper(ivt(p)))/ & + (SHR_CONST_PI * stocking(ivt(p)) * dwood(ivt(p))))**(1._r8/3._r8) else - htop(p) = ((3._r8 * deadstemc(p) * taper(p) * taper(p))/ & - (SHR_CONST_PI * stocking(p) * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * taper(ivt(p)) * taper(ivt(p)))/ & + (SHR_CONST_PI * stocking(ivt(p)) * dwood(ivt(p))))**(1._r8/3._r8) end if ! Peter Thornton, 5/3/2004 @@ -239,7 +236,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! Wang and Zeng, 2007. if (woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) - fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(p) * (htop(p)-hbot(p)))) ** vegshape(p) + fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(ivt(p)) * (htop(p)-hbot(p)))) ** vegshape(ivt(p)) else fb = 1._r8 - max(min(snow_depth(c),0.2_r8),0._r8)/0.2_r8 ! 0.2m is assumed !depth of snow required for complete burial of grasses diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index e433d1f305a..046bd880c0f 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -1093,12 +1093,25 @@ subroutine pftconrd if (.not. readv ) bendresist(:) = 1._r8 call ncd_io('vegshape', vegshape, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) vegshape(:) = 1._r8 + ! check validity + do i = 0, npft -1 + if (bendresist(i) .gt. 1.0_r8 .or. bendresist(i) .le. 0._r8) then + call endrun(msg="Non-physical selection of bendresist parameter, set between 0 and 1"//errMsg(__FILE__, __LINE__)) + end if + if (vegshape(i) .gt. 2.0_r8 .or. vegshape(i) .le. 0_r8) then + call endrun(msg="Non-physical selection of vegshape parameter, set between 0 and 2"//errMsg(__FILE__, __LINE__)) + end if + end do call ncd_io('stocking', stocking, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) stocking(:) = 1000._r8 + ! convert from stems/ha -> stems/m2 + do i = 0, npft + stocking(i) = stocking(i) / 10000._r8 + end do call ncd_io('taper', taper, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) then taper(:) = 200._r8 - do i = 0, npft - 1 + do i = 0, npft ! RPF 240331 - need to revisit this section when integrating with IM4, ! to make consistent with attempt to remove hard-coded pft numbers. if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) taper(i) = 10.0_r8 ! shrubs From 20bb484b9a491ae68e9fc5ccf00687417b29068f Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Thu, 22 Aug 2024 09:39:47 -0600 Subject: [PATCH 051/366] Rearrange taper assignment in pftvarcon The taper array was being initialized incorrectly, leading to developer test failures. This moves the definition down further to be able to initialize correctly. --- components/elm/src/main/pftvarcon.F90 | 31 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index 046bd880c0f..df732315d93 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -313,10 +313,11 @@ module pftvarcon real(r8), allocatable :: gcbr_q(:) !effectiveness of roots in reducing runoff-driven erosion ! NGEE Arctic snow-vegetation interactions - real(r8), allocatable :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011) + real(r8), allocatable :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011; Sturm et al. 2005) real(r8), allocatable :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) real(r8), allocatable :: stocking(:) ! stocking density for pft (stems / hectare) real(r8), allocatable :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) + logical :: taper_defaults ! set flag to use taper defaults if not on params file (necessary as import and set values are in different places) ! new pft properties, together with woody, crop, percrop, evergreen, stress_decid, season_decid, defined above, ! are introduced to define vegetation properties. This will be well defineing a pft so that no indices needed for codes. @@ -1094,7 +1095,7 @@ subroutine pftconrd call ncd_io('vegshape', vegshape, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) vegshape(:) = 1._r8 ! check validity - do i = 0, npft -1 + do i = 0, mxpft if (bendresist(i) .gt. 1.0_r8 .or. bendresist(i) .le. 0._r8) then call endrun(msg="Non-physical selection of bendresist parameter, set between 0 and 1"//errMsg(__FILE__, __LINE__)) end if @@ -1103,19 +1104,12 @@ subroutine pftconrd end if end do call ncd_io('stocking', stocking, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if (.not. readv ) stocking(:) = 1000._r8 - ! convert from stems/ha -> stems/m2 - do i = 0, npft - stocking(i) = stocking(i) / 10000._r8 - end do + if (.not. readv ) stocking(:) = 0.1_r8 ! convert previous default of 1000 stems/ha to stems/m2 as had been done in VegStructUpdateMod.F90 + taper_defaults = .false. call ncd_io('taper', taper, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) then - taper(:) = 200._r8 - do i = 0, npft - ! RPF 240331 - need to revisit this section when integrating with IM4, - ! to make consistent with attempt to remove hard-coded pft numbers. - if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) taper(i) = 10.0_r8 ! shrubs - end do + taper(:) = 200._r8 ! pftnames not set to integers yet, so reassign further down. + taper_defaults = .true. end if ! NOTE: the following 5 PFT flags/options are addtions to 'woody', 'stress_decid', 'season_decid', @@ -1548,6 +1542,17 @@ subroutine pftconrd end do end if + ! reassign taper values for shrubs - RPF + if (taper_defaults) then + do i = 0, mxpft + if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) then + taper(i) = 10._r8 ! shrubs + else + taper(i) = 200._r8 + endif + end do + end if + if (masterproc) then write(iulog,*) 'Successfully read PFT physiological data' write(iulog,*) From e979a2cb9bcf93db43dba21aad8c76beed1c50d9 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Thu, 22 Aug 2024 18:59:16 -0600 Subject: [PATCH 052/366] Add IM3 ERS developer test to e3sm_land_developer Adds a new exact restart test that uses new parameters in the param file for snow/vegetation interactions. --- cime_config/tests.py | 1 + .../testmods_dirs/elm/snowveg_arctic/shell_commands | 10 ++++++++++ .../testmods_dirs/elm/snowveg_arctic/user_nl_elm | 8 ++++++++ 3 files changed, 19 insertions(+) create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b8397..d4c10f126b2 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -93,6 +93,7 @@ "SMS_Ly2_P1x1.1x1_smallvilleIA.IELMCNCROP.elm-fan", "SMS.r05_r05.IELM.elm-topounit", "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-snowveg_arctic", "ERS.r05_r05.IELM.elm-lnd_rof_2way", "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands new file mode 100644 index 00000000000..396d8cea8d6 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands @@ -0,0 +1,10 @@ +./xmlchange LND_DOMAIN_FILE=domain_42_FLUXNETSITES_simyr1850_c170912.nc +./xmlchange ATM_DOMAIN_FILE=domain_42_FLUXNETSITES_simyr1850_c170912.nc +./xmlchange LND_DOMAIN_PATH="\$DIN_LOC_ROOT/share/domains/domain.clm" +./xmlchange ATM_DOMAIN_PATH="\$DIN_LOC_ROOT/share/domains/domain.clm" +./xmlchange DATM_MODE=CLM1PT +./xmlchange DATM_CLMNCEP_YR_START=2000 +./xmlchange DATM_CLMNCEP_YR_END=2000 +./xmlchange ELM_USRDAT_NAME=42_FLUXNETSITES +./xmlchange NTASKS=1 +./xmlchange NTHRDS=1 diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm new file mode 100644 index 00000000000..bd07678a134 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm @@ -0,0 +1,8 @@ +fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/surfdata_42_FLUXNETSITES_simyr1850_c170912.nc' +! this test has a parameter file with four new parameters defined for NGEE Arctic IM3 +! meant to improve snow-vegetation interactions. +! bendresist -> varies between 0 and 1, represents how much vegetation bends under snow loading +! vegshape -> suggested value 1 (parabolic), but 2 also used previously (Liston and Heimstra, 2011) +! taper -> deadstem height/radius ratio, moved from hardcoded in VegStructUpdateMod to pftvarcon +! stocking -> individual density on landscape (plants/m2), moved from hardcoded in VegStructUpdateMod to pftvarcon +paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/clm_params_ngeea-im3_c240822.nc' From f0c93cf0b51ec029f7c35bde688ec4894e8f102f Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Thu, 22 Aug 2024 21:59:09 -0600 Subject: [PATCH 053/366] Fix new IM3 paramter values in pftvarcon fixes an issue where using less than the maximum pfts set new parameters with non-physical values --- components/elm/src/main/pftvarcon.F90 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index df732315d93..c6d4ab470bb 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -1095,11 +1095,13 @@ subroutine pftconrd call ncd_io('vegshape', vegshape, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) vegshape(:) = 1._r8 ! check validity - do i = 0, mxpft + do i = 0, npft-1 if (bendresist(i) .gt. 1.0_r8 .or. bendresist(i) .le. 0._r8) then + write(iulog,*) "bendresist(i) is:", i, bendresist(i), npft call endrun(msg="Non-physical selection of bendresist parameter, set between 0 and 1"//errMsg(__FILE__, __LINE__)) end if - if (vegshape(i) .gt. 2.0_r8 .or. vegshape(i) .le. 0_r8) then + if (vegshape(i) .gt. 2.0_r8 .or. vegshape(i) .le. 0._r8) then + write(iulog,*) "vegshape(i) is:", i, vegshape(i), npft call endrun(msg="Non-physical selection of vegshape parameter, set between 0 and 2"//errMsg(__FILE__, __LINE__)) end if end do @@ -1544,7 +1546,7 @@ subroutine pftconrd ! reassign taper values for shrubs - RPF if (taper_defaults) then - do i = 0, mxpft + do i = 0, npft-1 if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) then taper(i) = 10._r8 ! shrubs else From 0f02df7bef612dda1933aeba3b09fb2d0f797505 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Tue, 27 Aug 2024 18:18:12 -0600 Subject: [PATCH 054/366] remove local ci test --- .github/workflows/ci-compile-test.yml | 56 --------------------------- 1 file changed, 56 deletions(-) delete mode 100644 .github/workflows/ci-compile-test.yml diff --git a/.github/workflows/ci-compile-test.yml b/.github/workflows/ci-compile-test.yml deleted file mode 100644 index 90e592630c6..00000000000 --- a/.github/workflows/ci-compile-test.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: CI -on: - push: - branches: - - '**' -jobs: - container-test-job: - runs-on: ubuntu-latest - container: - image: rfiorella/e3sm-dev:latest - steps: - - name: Check out the NGEE-Arctic-E3SM repo - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - submodules: true - - name: Setup cime and input datafiles - run: | - mkdir ~/.cime - cp /home/e3smuser/.cime/* /github/home/.cime/ - git clone https://github.com/rfiorella/pt-e3sm-inputdata /home/e3smuser/inputdata - cd /home/e3smuser/inputdata/lnd/clm2/firedata - tar -zxvf clmforc.Li_2012_hdm_0.5x0.5_AVHRR_simyr1850-2010_c130401.nc.tar.gz - cd /home/e3smuser/inputdata/atm/datm7/NASA_LIS/ - tar -zxvf clmforc.Li_2012_climo1995-2011.T62.lnfm_c130327.nc.tar.gz - tar -zxvf clmforc.Li_2012_climo1995-2011.T62.lnfm_Total_c140423.nc.tar.gz - ls -v /home/e3smuser/inputdata/atm/datm7/NASA_LIS/ - - name: create new case and setup - run: | - cd cime/scripts && ./create_newcase --mach docker --res ELM_USRDAT --compset ICB1850CNPRDCTCBC --case ~/output/ci_test \ - && cd ~/output/ci_test - ./xmlchange MOSART_MODE=NULL,DOUT_S=FALSE,DIN_LOC_ROOT=/home/e3smuser/inputdata - ./xmlchange DIN_LOC_ROOT_CLMFORC=/home/e3smuser/inputdata/atm/datm7 - ./xmlchange ELM_USRDAT_NAME=1x1pt_AK-TLG - ./xmlchange ATM_DOMAIN_PATH=/home/e3smuser/inputdata/share/domains/domain.clm - ./xmlchange LND_DOMAIN_PATH=/home/e3smuser/inputdata/share/domains/domain.clm - ./xmlchange ATM_DOMAIN_FILE=domain.lnd.1x1pt_teller-GRID_navy.nc - ./xmlchange LND_DOMAIN_FILE=domain.lnd.1x1pt_teller-GRID_navy.nc - ./xmlchange PIO_TYPENAME=netcdf,NTASKS=1,NTHRDS=1 - ./xmlchange STOP_OPTION=nyears,STOP_N=5 - ./xmlchange BATCH_SYSTEM=none - echo "&clm_inparm" >> user_nl_elm - echo " do_budgets = .false." >> user_nl_elm - echo " use_nofire = .true." >> user_nl_elm - echo " metdata_type = 'gswp3'" >> user_nl_elm - echo " metdata_bypass = '/home/e3smuser/inputdata/atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v2.c180716_NGEE-Grid/cpl_bypass_teller-Grid'" >> user_nl_elm - echo " fsurdat = '/home/e3smuser/inputdata/lnd/clm2/surfdata_map/surfdata_1x1pt_teller-GRID_simyr1850_c360x720_c171002.nc'" >> user_nl_elm - echo " stream_fldfilename_popdens = '/home/e3smuser/inputdata/lnd/clm2/firedata/clmforc.Li_2012_hdm_0.5x0.5_AVHRR_simyr1850-2010_c130401.nc'" >> user_nl_elm - ./case.setup - # add coupler bypass flag to cmake macros. - echo 'string(APPEND CPPDEFS " -DCPL_BYPASS")' >> cmake_macros/universal.cmake - ./case.build || cat /home/e3smuser/output/ci_test/bld/e3sm.bldlog.* - ./case.submit --no-batch - cat /home/e3smuser/output/ci_test/run/e3sm.log.* - cat /home/e3smuser/output/ci_test/run/lnd.log.* - From 01ac9bdd8420c86eda51544c87f447d5db05d4c5 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Tue, 3 Sep 2024 10:23:15 -0600 Subject: [PATCH 055/366] Update comment to correctly reflect references in VegStructUpdateMod --- components/elm/src/biogeochem/VegStructUpdateMod.F90 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 868c417ff5f..74e135f6f0e 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -233,8 +233,10 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! adjust lai and sai for burying by snow. ! snow burial fraction for short vegetation (e.g. grasses) as in - ! Wang and Zeng, 2007. - if (woody(ivt(p)) >= 1.0_r8 ) then + ! Wang and Zeng et al 2007. + ! Taller vegetation (trees and shrubs) have been updated to use formulation similar to + ! Sturm et al. 2005; Liston and Hiemstra, 2011; and Belke-Brea et al. 2020 + if ( woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(ivt(p)) * (htop(p)-hbot(p)))) ** vegshape(ivt(p)) else From 3734c0fb2553621d30ee1f4cb8b8ab893ddb975a Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Fri, 6 Sep 2024 15:40:36 -0600 Subject: [PATCH 056/366] use woody instead of hardcoding to pft number --- components/elm/src/main/pftvarcon.F90 | 36 +++++++++++++-------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index c6d4ab470bb..66c2adda90f 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -459,18 +459,18 @@ subroutine pftconrd ! but cannot be less, i.e. there cannot be more pfts in the surface file than there are set of pft parameters, if running the non-crop version of model. if (npft= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) then + if (woody(i) == 2._r8) then taper(i) = 10._r8 ! shrubs - else + else if (woody(i) == 1._r8) then taper(i) = 200._r8 - endif + end if end do end if From 18fa418c5e3d132de7967c75fad67ffe8f75a56e Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Fri, 6 Sep 2024 15:57:46 -0600 Subject: [PATCH 057/366] update woody comment description to reflect IM4 changes --- .../elm/src/biogeochem/AllocationMod.F90 | 276 +++++++++--------- .../src/biogeochem/CNAllocationBetrMod.F90 | 12 +- .../src/biogeochem/CNGapMortalityBeTRMod.F90 | 8 +- .../src/biogeochem/CNNStateUpdate1BeTRMod.F90 | 2 +- .../elm/src/biogeochem/CNPhenologyBeTRMod.F90 | 8 +- .../src/biogeochem/CarbonStateUpdate1Mod.F90 | 2 +- .../elm/src/biogeochem/GrowthRespMod.F90 | 4 +- .../elm/src/biogeochem/MaintenanceRespMod.F90 | 6 +- .../biogeochem/NitrogenStateUpdate1Mod.F90 | 10 +- .../src/biogeochem/PhenologyFluxLimitMod.F90 | 14 +- .../elm/src/biogeochem/PhenologyMod.F90 | 32 +- .../biogeochem/PhosphorusStateUpdate1Mod.F90 | 12 +- .../data_types/VegetationPropertiesType.F90 | 4 +- 13 files changed, 195 insertions(+), 195 deletions(-) diff --git a/components/elm/src/biogeochem/AllocationMod.F90 b/components/elm/src/biogeochem/AllocationMod.F90 index 141194397fb..5c315fc1769 100644 --- a/components/elm/src/biogeochem/AllocationMod.F90 +++ b/components/elm/src/biogeochem/AllocationMod.F90 @@ -36,11 +36,11 @@ module AllocationMod use elm_varctl , only : NFIX_PTASE_plant use ELMFatesInterfaceMod , only : hlm_fates_interface_type use elm_varctl , only: iulog - use elm_varctl , only : carbon_only - use elm_varctl , only : carbonnitrogen_only + use elm_varctl , only : carbon_only + use elm_varctl , only : carbonnitrogen_only use elm_varctl , only : carbonphosphorus_only use shr_infnan_mod , only: nan => shr_infnan_nan, assignment(=) - + ! implicit none save @@ -114,23 +114,23 @@ module AllocationMod ! to toggle and update which processes are active. ! This will get set to false ! after ad_carbon_only is complete. - - + + logical :: crop_supln = .false. !Prognostic crop receives supplemental Nitrogen - + real(r8), allocatable,target :: veg_rootc_bigleaf(:,:) ! column-level fine-root biomas kgc/m3 integer, pointer :: ft_index_bigleaf(:) ! array holding the pft index of each competitor ! ECA parameters - ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance - real(r8), parameter :: e_plant_scalar = 0.0000125_r8 - - ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance - real(r8), parameter :: e_decomp_scalar = 0.05_r8 + ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance + real(r8), parameter :: e_plant_scalar = 0.0000125_r8 + + ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance + real(r8), parameter :: e_decomp_scalar = 0.05_r8 !$acc declare create(e_decomp_scalar) !$acc declare create(e_plant_scalar) - + !$acc declare copyin(crop_supln) !----------------------------------------------------------------------- @@ -213,8 +213,8 @@ subroutine AllocationInit ( bounds, elm_fates) use elm_time_manager, only: get_step_size use elm_varpar , only: crop_prog use elm_varctl , only: iulog - use elm_varctl , only : carbon_only - use elm_varctl , only : carbonnitrogen_only + use elm_varctl , only : carbon_only + use elm_varctl , only : carbonnitrogen_only use elm_varctl , only : carbonphosphorus_only @@ -234,9 +234,9 @@ subroutine AllocationInit ( bounds, elm_fates) integer :: max_comps ! maximum number of possible plant competitors ! elm big-leaf: number of pfts/patches ! fates: number of cohorts in the column - - + + !----------------------------------------------------------------------- if ( crop_prog )then @@ -253,7 +253,7 @@ subroutine AllocationInit ( bounds, elm_fates) end if end if - + ! set time steps dt = real( get_step_size(), r8 ) @@ -374,7 +374,7 @@ subroutine EvaluateSupplStatus() end subroutine EvaluateSupplStatus !------------------------------------------------------------------------------------------------- - + subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp, filter_soilp, & photosyns_vars, crop_vars, canopystate_vars, cnstate_vars, dt, yr) ! PHASE-1 of Allocation: loop over patches to assess the total plant N demand and P demand @@ -430,7 +430,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -971,7 +971,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Fractional uptake profiles, that are proportional to root density real(r8):: nuptake_prof(bounds%begc:bounds%endc,1:nlevdecomp) real(r8):: puptake_prof(bounds%begc:bounds%endc,1:nlevdecomp) - integer, allocatable :: filter_pcomp(:) ! this is a plant competitor map for FATES/ELM-BL w/ ECA + integer, allocatable :: filter_pcomp(:) ! this is a plant competitor map for FATES/ELM-BL w/ ECA real(r8), allocatable,target :: plant_nh4demand_vr_fates(:,:) ! nh4 demand per competitor per soil layer real(r8), allocatable,target :: plant_no3demand_vr_fates(:,:) ! no3 demand per competitor per soil layer real(r8), allocatable,target :: plant_pdemand_vr_fates(:,:) ! p demand per competitor per soil layer @@ -990,7 +990,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & real(r8):: cp_stoich_var=0.4 ! variability of CP ratio - + !----------------------------------------------------------------------- associate( & @@ -1152,7 +1152,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & n_pcomp = elm_fates%fates(ci)%bc_out(s)%num_plant_comps pci = 1 pcf = n_pcomp - + if( nu_com.eq.'RD') then ! Overwrite the column level demands, since fates plants are all sharing @@ -1160,13 +1160,13 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! to scale up to column plant_ndemand_col(c) = 0._r8 plant_pdemand_col(c) = 0._r8 - + ! We fill the vertically resolved array to simplify some jointly used code do j = 1, nlevdecomp col_plant_ndemand_vr(c,j) = 0._r8 col_plant_pdemand_vr(c,j) = 0._r8 - + do f = 1,n_pcomp ft = elm_fates%fates(ci)%bc_out(s)%ft_index(f) @@ -1184,10 +1184,10 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! [gN/m2/s] plant_ndemand_col(c) = plant_ndemand_col(c) + col_plant_ndemand_vr(c,j)*dzsoi_decomp(j) plant_pdemand_col(c) = plant_pdemand_col(c) + col_plant_pdemand_vr(c,j)*dzsoi_decomp(j) - + end do - + else !(ECA) do f = 1,n_pcomp @@ -1195,7 +1195,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do veg_rootc_ptr => elm_fates%fates(ci)%bc_out(s)%veg_rootc - ft_index_ptr => elm_fates%fates(ci)%bc_out(s)%ft_index ! Should be + ft_index_ptr => elm_fates%fates(ci)%bc_out(s)%ft_index ! Should be decompmicc(:) = elm_fates%fates(ci)%bc_out(s)%decompmicc(:) ! Should be (nlevdecomp) cn_scalar_runmean_ptr => elm_fates%fates(ci)%bc_out(s)%cn_scalar ! This is 1.0 @@ -1216,7 +1216,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & pci = col_pp%pfti(c) ! Initial plant competitor index pcf = col_pp%pftf(c) ! Final plant competitor index - + if (nu_com .eq. 'RD') then do j = 1, nlevdecomp @@ -1225,7 +1225,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do else - + f = 0 decompmicc(:) = 0._r8 do p = col_pp%pfti(c), col_pp%pftf(c) @@ -1246,7 +1246,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end if end do n_pcomp = f - + ft_index_ptr => ft_index_bigleaf veg_rootc_ptr => veg_rootc_bigleaf @@ -1263,7 +1263,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & (leafcn(ivt(p)) - leafcn(ivt(p))*(1- cn_stoich_var)),0.0_r8),1.0_r8) end do end if - + km_nh4_ptr => km_plant_nh4 vmax_nh4_ptr => vmax_plant_nh4 cn_scalar_runmean_ptr => cn_scalar_runmean @@ -1298,14 +1298,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! (1) add nitrogen and phosphorus competition ! (2) nitrogen and phosphorus uptake is based on root kinetics ! (3) no second pass nutrient uptake for plants - ! ============================================================= - + ! ============================================================= + if (nu_com .eq. 'RD') then ! Estimate actual allocation rates via Relative Demand ! approach (RD) - + call NAllocationRD(col_plant_ndemand_vr(c,:), & ! IN potential_immob_vr(c,:), & ! IN AllocParamsInst%compet_plant_nh4, & ! IN @@ -1332,19 +1332,19 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Estimate actual allocation rates via Capacitance Aquisition ! approach (ECA/CA) - + call NAllocationECAMIC(pci,dt, & ! IN bd(c,:), & ! IN h2osoi_vol(c,:), & ! IN t_scalar(c,:), & ! IN - n_pcomp, & ! IN + n_pcomp, & ! IN filter_pcomp(1:n_pcomp), & ! IN veg_rootc_ptr(pci:pcf,:), & ! IN ft_index_ptr(pci:pcf), & ! IN cn_scalar_runmean_ptr(pci:pcf), & ! IN decompmicc, & ! IN smin_nh4_vr(c,:), & ! IN - nu_com, & ! IN + nu_com, & ! IN km_nh4_ptr, & ! IN vmax_nh4_ptr, & ! IN km_decomp_nh4, & ! IN @@ -1389,7 +1389,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! NO3 flux demands. supplement_to_sminn_vr(c,j) = 0._r8 if (carbon_only .or. carbonphosphorus_only) then - + if ( fpi_no3_vr(j) + fpi_nh4_vr(j) < 1._r8 ) then fpi_vr(c,j) = 1._r8 fpi_nh4_vr(j) = 1.0_r8 - fpi_no3_vr(j) @@ -1409,9 +1409,9 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & supplement_to_sminn_vr(c,j) = supplement_to_sminn_vr(c,j) + col_plant_ndemand_vr(c,j) smin_nh4_to_plant_vr(c,j) = col_plant_ndemand_vr(c,j) - smin_no3_to_plant_vr(c,j) end if - - + + end if ! sum up nitrogen limitation to decomposition @@ -1422,14 +1422,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_vr(c,j) = actual_immob_no3_vr(c,j) + actual_immob_nh4_vr(c,j) end do - + ! Starting resolving P limitation !!! ! ============================================================= if (nu_com .eq. 'RD') then ! Relative Demand (RD) - + call PAllocationRD(col_plant_pdemand_vr(c,:), & ! IN potential_immob_p_vr(c,:), & ! IN solutionp_vr(c,:), & ! IN @@ -1438,38 +1438,38 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_p_vr(c,:), & ! OUT sminp_to_plant_vr(c,:), & ! OUT supplement_to_sminp_vr(c,:)) ! OUT - + else call PAllocationECAMIC(pci,dt, & ! IN h2osoi_vol(c,:), & ! IN - t_scalar(c,:), & ! IN - gross_pmin_vr(c,:), & ! IN - potential_immob_p_vr(c,:), & ! IN - biochem_pmin_vr_col(c,:), & ! IN - primp_to_labilep_vr_col(c,:), & ! IN - pdep_to_sminp(c), & ! IN - pdep_prof(c,:), & ! IN - vmax_minsurf_p_vr(isoilorder(c),:), & ! IN - km_minsurf_p_vr(isoilorder(c),:), & ! IN - solutionp_vr(c,:), & ! IN - nu_com, & ! IN - n_pcomp, & ! IN + t_scalar(c,:), & ! IN + gross_pmin_vr(c,:), & ! IN + potential_immob_p_vr(c,:), & ! IN + biochem_pmin_vr_col(c,:), & ! IN + primp_to_labilep_vr_col(c,:), & ! IN + pdep_to_sminp(c), & ! IN + pdep_prof(c,:), & ! IN + vmax_minsurf_p_vr(isoilorder(c),:), & ! IN + km_minsurf_p_vr(isoilorder(c),:), & ! IN + solutionp_vr(c,:), & ! IN + nu_com, & ! IN + n_pcomp, & ! IN filter_pcomp(1:n_pcomp), & ! IN - veg_rootc_ptr(pci:pcf,:), & ! IN - ft_index_ptr(pci:pcf), & ! IN - decompmicc, & ! IN - cp_scalar_runmean_ptr(pci:pcf), & ! IN - km_p_ptr(:), & ! IN - vmax_p_ptr(:), & ! IN - km_decomp_p, & ! IN - labilep_vr(c,:), & ! IN + veg_rootc_ptr(pci:pcf,:), & ! IN + ft_index_ptr(pci:pcf), & ! IN + decompmicc, & ! IN + cp_scalar_runmean_ptr(pci:pcf), & ! IN + km_p_ptr(:), & ! IN + vmax_p_ptr(:), & ! IN + km_decomp_p, & ! IN + labilep_vr(c,:), & ! IN plant_pdemand_vr_ptr(pci:pcf,:), & ! INOUT - col_plant_pdemand_vr(c,:), & ! OUT + col_plant_pdemand_vr(c,:), & ! OUT adsorb_to_labilep_vr(c,:), & ! OUT fpi_p_vr(c,:), & ! OUT actual_immob_p_vr(c,:), & ! OUT - sminp_to_plant_vr(c,:), & ! OUT + sminp_to_plant_vr(c,:), & ! OUT desorb_to_solutionp_vr(c,:), & ! OUT supplement_to_sminp_vr(c,:)) ! OUT @@ -1482,13 +1482,13 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & supplement_to_sminp_vr(c,j) = col_plant_pdemand_vr(c,j) end do end if - + end if ! end of P competition ! resolving N limitation vs. P limitation for decomposition ! update (1) actual immobilization for N and P (2) sminn_to_plant and sminp_to_plant ! We only resolve co-limitations when are supplementing neither element - + np_bothactive: if ( .not.carbon_only .and. & .not.carbonphosphorus_only .and. & .not.carbonnitrogen_only ) then @@ -1593,7 +1593,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_vr(c,j) = potential_immob_vr(c,j) * fpi_p_vr(c,j) end do end if - + ! sum up plant N/P uptake at column level and patch level ! sum up N fluxes to plant after initial competition sminn_to_plant(c) = 0._r8 @@ -1602,7 +1602,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & sminn_to_plant(c) = sminn_to_plant(c) + sminn_to_plant_vr(c,j) * dzsoi_decomp(j) sminp_to_plant(c) = sminp_to_plant(c) + sminp_to_plant_vr(c,j) * dzsoi_decomp(j) end do - + ! update column plant N/P demand, pft level plant NP uptake for ECA and MIC mode eca_filter: if (nu_com .eq. 'ECA' .or. nu_com .eq. 'MIC') then @@ -1648,14 +1648,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end if end do end if - + end do end if eca_filter end do col_loop - + if ((nu_com .eq. 'ECA' .or. nu_com .eq. 'MIC')) then deallocate(filter_pcomp) if(.not.use_fates)then @@ -1750,7 +1750,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Set the FATES N and P uptake fluxes - + if(use_fates)then do fc=1,num_soilc c = filter_soilc(fc) @@ -1771,15 +1771,15 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & (elm_fates%fates(ci)%bc_pconst%vmax_nh4(ft)+elm_fates%fates(ci)%bc_pconst%vmax_no3(ft)) * & dzsoi_decomp(j) end do - + do j = 1,nlevdecomp - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & smin_nh4_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (ndemand/plant_ndemand_col(c)) - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & smin_no3_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (ndemand/plant_ndemand_col(c)) @@ -1787,12 +1787,12 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do end do end if - + if( plant_pdemand_col(c)>tiny(plant_pdemand_col(c)) ) then do f = 1,n_pcomp ft = elm_fates%fates(ci)%bc_out(s)%ft_index(f) - + pdemand=0._r8 do j = 1,nlevdecomp ! [gP/m2/s] @@ -1800,14 +1800,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & elm_fates%fates(ci)%bc_pconst%vmax_p(ft) * & dzsoi_decomp(j) end do - + do j = 1,nlevdecomp ! [gP/m2/step] - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & sminp_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (pdemand/plant_pdemand_col(c)) - + end do end do end if @@ -1817,21 +1817,21 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & do f = 1,n_pcomp do j = 1,nlevdecomp - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & plant_nh4demand_vr_fates(f,j) * fpg_nh4_vr(c,j) * dzsoi_decomp(j) * dt - - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & + + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & plant_no3demand_vr_fates(f,j) * fpg_no3_vr(c,j) * dzsoi_decomp(j) * dt - - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & + + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & (plant_pdemand_vr_fates(f,j) * fpg_p_vr(c,j)) * dzsoi_decomp(j) * dt - + end do end do - + end if end do @@ -1840,7 +1840,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & deallocate(plant_no3demand_vr_fates) deallocate(plant_pdemand_vr_fates) end if - + end if ! if(use_fates) end associate @@ -1907,7 +1907,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -2959,7 +2959,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & end associate end subroutine Allocation3_PlantCNPAlloc - + ! ====================================================================================== subroutine NAllocationECAMIC(pci,dt, & ! IN @@ -2976,14 +2976,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN nu_com, & ! IN km_nh4_plant, & ! IN (pft) vmax_nh4_plant, & ! IN (pft) - km_decomp_nh4, & ! IN + km_decomp_nh4, & ! IN potential_immob_vr, & ! IN (j) plant_nh4demand_vr, & ! INOUT (i,j) col_plant_nh4demand_vr, & ! OUT (j) fpi_nh4_vr, & ! OUT (j) actual_immob_nh4_vr, & ! OUT (j) smin_nh4_to_plant_vr, & ! OUT (j) - smin_no3_vr, & ! IN (j) + smin_no3_vr, & ! IN (j) km_no3_plant, & ! IN (pft) vmax_no3_plant, & ! IN (pft) km_decomp_no3, & ! IN (j) @@ -3004,12 +3004,12 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ! kinetics following Zhu et al., 2016 DOI: 10.1002/2016JG003554 ! ------------------------------------------------------------------------------------ use elm_varpar , only: nlevdecomp - + integer, intent(in) :: pci ! First index of plant comp arrays real(r8), intent(in) :: dt ! Time step duration [s] real(r8), intent(in) :: bd(:) ! Bulk density of dry soil material [kg m-3] real(r8), intent(in) :: h2osoi_vol(:) ! Vol. Soil Water in each layer [m3] - real(r8), intent(in) :: t_scalar(:) ! fraction by which decomposition is limited by temperature + real(r8), intent(in) :: t_scalar(:) ! fraction by which decomposition is limited by temperature integer, intent(in) :: n_pcomp ! number of plant competitors integer, intent(in) :: filter_pcomp(:) ! plant competition filter real(r8), intent(in) :: veg_rootc(pci:,:) ! total fine-root biomass of each competitor [gC/m3] @@ -3065,7 +3065,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN integer :: i,ip ! loop index for competitors integer :: ft ! loop index for pfts - ! 2.76 consider soil adsorption effect on [NH4+] availability, + ! 2.76 consider soil adsorption effect on [NH4+] availability, ! based on Zhu et al., 2016 DOI: 10.1002/2016JG003554 real(r8), parameter :: adsorp_nh4_eff = 2.76_r8 @@ -3073,14 +3073,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN do j = 1, nlevdecomp - ! Plant, microbial decomposers compete for NH4. Thus loop over each + ! Plant, microbial decomposers compete for NH4. Thus loop over each ! plant competitor in this competitive space (column). - ! Calculate competition coefficients for N/P, first need to convert - ! concentration to per soil water based + ! Calculate competition coefficients for N/P, first need to convert + ! concentration to per soil water based ! concentration of mineralized nutrient, per soil water solution_conc = smin_nh4_vr(j) / (bd(j)*adsorp_nh4_eff*m3_per_liter + h2osoi_vol(j)) - + e_km = 0._r8 do i = 1, n_pcomp ip = filter_pcomp(i) @@ -3089,14 +3089,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN end do e_km = e_km + e_decomp_scalar*decompmicc(j)*(1._r8/km_decomp_nh4 + 1._r8/km_nit) - + do i = 1, n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_conc / & + compet_plant(i) = solution_conc / & ( km_nh4_plant(ft) * (1._r8 + solution_conc/km_nh4_plant(ft) + e_km)) end do - + compet_decomp = solution_conc / (km_decomp_nh4 * (1._r8 + solution_conc/km_decomp_nh4 + e_km)) compet_nit = solution_conc / (km_nit * (1._r8 + solution_conc/km_nit + e_km)) @@ -3110,7 +3110,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ip = filter_pcomp(i) ft = ft_index(ip) - ! This is the demand per m3 of the column (not patch) + ! This is the demand per m3 of the column (not patch) ! (for native ELM divide through by the patch weight to get per m3 of patch) plant_nh4demand_vr(ip,j) = max(0._r8,vmax_nh4_plant(ft) * veg_rootc(ip,j) * & cn_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) @@ -3130,7 +3130,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN else ! 'MIC' mode - sum_nh4_demand_scaled = potential_immob_vr(j)*compet_decomp + & + sum_nh4_demand_scaled = potential_immob_vr(j)*compet_decomp + & pot_f_nit_vr(j)*compet_nit end if @@ -3199,10 +3199,10 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN do i = 1, n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_conc / & + compet_plant(i) = solution_conc / & ( km_no3_plant(ft) * (1._r8 + solution_conc/km_no3_plant(ft) + e_km)) end do - + compet_decomp = solution_conc / (km_decomp_no3 * (1._r8 + solution_conc/km_decomp_no3 + e_km)) compet_denit = solution_conc / (km_den * (1._r8 + solution_conc/km_den + e_km)) @@ -3215,7 +3215,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ip = filter_pcomp(i) ft = ft_index(ip) - ! This is the demand per m3 of the column (not patch) + ! This is the demand per m3 of the column (not patch) ! (for native ELM divide through by the patch weight to get per m3 of patch) plant_no3demand_vr(ip,j) = max(0._r8,vmax_no3_plant(ft) * veg_rootc(ip,j) * & cn_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) @@ -3245,7 +3245,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN smin_no3_to_plant_vr(j) = col_plant_no3demand_vr(j) f_denit_vr(j) = pot_f_denit_vr(j) - else + else ! NO3 availability can not satisfy the sum of immobilization, denitrification, and ! plant growth demands, so these three demands compete for available @@ -3284,34 +3284,34 @@ end subroutine NAllocationECAMIC subroutine PAllocationECAMIC(pci, & dt, & - h2osoi_vol, & - t_scalar, & - gross_pmin_vr, & - potential_immob_p_vr, & - biochem_pmin_vr_col, & - primp_to_labilep_vr_col, & - pdep_to_sminp, & - pdep_prof, & + h2osoi_vol, & + t_scalar, & + gross_pmin_vr, & + potential_immob_p_vr, & + biochem_pmin_vr_col, & + primp_to_labilep_vr_col, & + pdep_to_sminp, & + pdep_prof, & vmax_minsurf_p_vr, & km_minsurf_p_vr, & solutionp_vr, & nu_com, & n_pcomp, & - filter_pcomp, & - veg_rootc, & + filter_pcomp, & + veg_rootc, & ft_index, & decompmicc, & - cp_scalar_runmean, & + cp_scalar_runmean, & km_plant_p, & vmax_plant_p, & - km_decomp_p, & + km_decomp_p, & labilep_vr, & - plant_pdemand_vr_patch, & - col_plant_pdemand_vr, & + plant_pdemand_vr_patch, & + col_plant_pdemand_vr, & adsorb_to_labilep_vr, & fpi_p_vr, & actual_immob_p_vr, & - sminp_to_plant_vr, & + sminp_to_plant_vr, & desorb_to_solutionp_vr, & supplement_to_sminp_vr) @@ -3370,7 +3370,7 @@ subroutine PAllocationECAMIC(pci, & ! plant P uptake, microbial P uptake/release ! secondary P desorption is assumed to go into solution P pool - do j = 1, nlevdecomp + do j = 1, nlevdecomp ! plant, microbial decomposer, mineral surface compete for P ! loop over each pft within the same column @@ -3390,27 +3390,27 @@ subroutine PAllocationECAMIC(pci, & do i = 1,n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_pconc / & + compet_plant(i) = solution_pconc / & (km_plant_p(ft)*(1._r8 + solution_pconc/km_plant_p(ft) + e_km_p)) end do - + compet_decomp_p = solution_pconc / & (km_decomp_p * (1._r8 + solution_pconc/km_decomp_p + e_km_p)) - compet_minsurf_p = solution_pconc/ & + compet_minsurf_p = solution_pconc/ & (km_minsurf_p_vr(j) * (1._r8 + solution_pconc/km_minsurf_p_vr(j) + e_km_p)) col_plant_pdemand_vr(j) = 0._r8 do i = 1,n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - plant_pdemand_vr_patch(ip,j) = max(0._r8,vmax_plant_p(ft) * veg_rootc(ip,j) * & + plant_pdemand_vr_patch(ip,j) = max(0._r8,vmax_plant_p(ft) * veg_rootc(ip,j) * & cp_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) col_plant_pdemand_vr(j) = col_plant_pdemand_vr(j) + plant_pdemand_vr_patch(ip,j) end do ! potential adsorption rate without plant and microbial interaction - ! including weathering, deposition, phosphatase, mineralization, + ! including weathering, deposition, phosphatase, mineralization, ! immobilization, plant uptake dsolutionp_dt = gross_pmin_vr(j) -potential_immob_p_vr(j) - & col_plant_pdemand_vr(j) + biochem_pmin_vr_col(j) + & @@ -3477,7 +3477,7 @@ subroutine PAllocationECAMIC(pci, & if (nu_com .eq. 'MIC') sminp_to_plant_vr(j) = min(max( 0._r8, & (solutionp_vr(j)/dt) - actual_immob_p_vr(j) - adsorb_to_labilep_vr(j) ), & - col_plant_pdemand_vr(j)) + col_plant_pdemand_vr(j)) end if end do @@ -3488,7 +3488,7 @@ end subroutine PAllocationECAMIC subroutine NAllocationRD(col_plant_ndemand_vr, &! IN (j) potential_immob_vr, & ! IN (j) - compet_plants_nh4, & ! IN + compet_plants_nh4, & ! IN compet_decomp_nh4, & ! IN dt, & ! IN smin_nh4_vr, & ! IN (j) @@ -3641,7 +3641,7 @@ end subroutine NAllocationRD ! ====================================================================================== - subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN + subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN potential_immob_p_vr, & ! IN (j) solutionp_vr, & ! IN (j) dt, & ! IN @@ -3684,7 +3684,7 @@ subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN actual_immob_p_vr(j) = potential_immob_p_vr(j) sminp_to_plant_vr(j) = col_plant_pdemand_vr(j) supplement_to_sminp_vr(j) = sum_pdemand - (solutionp_vr(j)/dt) - + else ! P availability can not satisfy the sum of immobilization and ! plant growth demands, so these two demands compete for @@ -3702,7 +3702,7 @@ subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN fpi_p_vr(j) = 0.0_r8 end if - sminp_to_plant_vr(j) = max( 0._r8,(solutionp_vr(j)/dt) - actual_immob_p_vr(j) ) + sminp_to_plant_vr(j) = max( 0._r8,(solutionp_vr(j)/dt) - actual_immob_p_vr(j) ) end if end do @@ -3749,7 +3749,7 @@ subroutine calc_nuptake_prof(bounds, num_soilc, filter_soilc, cnstate_vars, nupt c = filter_soilc(fc) sminn_vr_loc(c,j) = smin_no3_vr(c,j) + smin_nh4_vr(c,j) - + if(use_pflotran .and. pf_cmode) then sminn_tot(c) = sminn_tot(c) + sminn_vr_loc(c,j) * dzsoi_decomp(j) & *(nfixation_prof(c,j)*dzsoi_decomp(j)) ! weighted by froot fractions in annual max. active layers diff --git a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 index 8cfc342bb40..32f9fc6eb23 100644 --- a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 +++ b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 @@ -26,13 +26,13 @@ module CNAllocationBeTRMod use PhotosynthesisType , only : photosyns_type use CropType , only : crop_type use VegetationPropertiesType, only : veg_vp - use LandunitType , only : lun_pp + use LandunitType , only : lun_pp use ColumnType , only : col_pp - use ColumnDataType , only : col_cf, col_ns, col_nf, col_ps, col_pf + use ColumnDataType , only : col_cf, col_ns, col_nf, col_ps, col_pf use VegetationType , only : veg_pp use VegetationDataType , only : veg_cs, veg_ns, veg_nf, veg_ps, veg_pf - use VegetationDataType , only : veg_cf, c13_veg_cf, c14_veg_cf - + use VegetationDataType , only : veg_cf, c13_veg_cf, c14_veg_cf + ! bgc interface & pflotran module switches use elm_varctl , only : nu_com use SoilStatetype , only : soilstate_type @@ -359,7 +359,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -1167,7 +1167,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type ! - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) diff --git a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 index 3bbbf1db1f6..ed8811f61f8 100644 --- a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 @@ -18,9 +18,9 @@ module CNGapMortalityBeTRMod use ColumnType , only : col_pp use ColumnDataType , only : col_cf, col_nf, col_pf use VegetationPropertiesType , only : veg_vp - use VegetationType , only : veg_pp - use VegetationDataType , only : veg_cs, veg_cf, veg_ns, veg_nf - use VegetationDataType , only : veg_ps, veg_pf + use VegetationType , only : veg_pp + use VegetationDataType , only : veg_cs, veg_cf, veg_ns, veg_nf + use VegetationDataType , only : veg_ps, veg_pf use PhosphorusFluxType , only : phosphorusflux_type use PhosphorusStateType , only : phosphorusstate_type @@ -121,7 +121,7 @@ subroutine CNGapMortality (& associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody & ! Input: [real(r8) (:) ] binary flag for woody lifeform + woody => veg_vp%woody & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) ) ! set the mortality rate based on annual rate diff --git a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 index 465c92f82b4..b785f52549e 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 @@ -59,7 +59,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step diff --git a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 index ac3b58538e8..066dcfb62c8 100644 --- a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 @@ -525,7 +525,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp , & prev_dayl => grc_pp%prev_dayl , & ! Input: [real(r8) (:) ] daylength from previous time step (s) season_decid => veg_vp%season_decid , & ! Input: [real(r8) (:) ] binary flag for seasonal-deciduous leaf habit (0 or 1) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) t_soisno => col_es%t_soisno , & ! Input: [real(r8) (:,:) ] soil temperature (Kelvin) (-nlevsno+1:nlevgrnd) @@ -875,7 +875,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & dayl => grc_pp%dayl , & ! Input: [real(r8) (:) ] daylength (s) leaf_long => veg_vp%leaf_long , & ! Input: [real(r8) (:) ] leaf longevity (yrs) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) stress_decid => veg_vp%stress_decid , & ! Input: [real(r8) (:) ] binary flag for stress-deciduous leaf habit (0 or 1) soilpsi => soilstate_vars%soilpsi_col , & ! Input: [real(r8) (:,:) ] soil water potential in each soil layer (MPa) @@ -2295,7 +2295,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) onset_flag => cnstate_vars%onset_flag_patch , & ! Input: [real(r8) (:) ] onset flag onset_counter => cnstate_vars%onset_counter_patch , & ! Input: [real(r8) (:) ] onset days counter @@ -2880,7 +2880,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) livewdcn => veg_vp%livewdcn , & ! Input: [real(r8) (:) ] live wood (phloem and ray parenchyma) C:N (gC/gN) deadwdcn => veg_vp%deadwdcn , & ! Input: [real(r8) (:) ] dead wood (xylem and heartwood) C:N (gC/gN) diff --git a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 index 1eb7f64d025..5133575abd3 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 @@ -200,7 +200,7 @@ subroutine CarbonStateUpdate1(bounds, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step harvdate => crop_vars%harvdate_patch & ! Input: [integer (:) ] harvest date diff --git a/components/elm/src/biogeochem/GrowthRespMod.F90 b/components/elm/src/biogeochem/GrowthRespMod.F90 index cb7ad522265..cc6707d723b 100644 --- a/components/elm/src/biogeochem/GrowthRespMod.F90 +++ b/components/elm/src/biogeochem/GrowthRespMod.F90 @@ -51,7 +51,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) associate( & ivt => veg_pp%itype , & ! Input: [integer (:)] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:)] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:)] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cpool_to_leafc => veg_cf%cpool_to_leafc , & ! Input: [real(r8) (:)] cpool_to_leafc_storage => veg_cf%cpool_to_leafc_storage , & ! Input: [real(r8) (:)] @@ -98,7 +98,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) ) ! Loop through patches - ! start pft loop + ! start pft loop do fp = 1,num_soilp p = filter_soilp(fp) diff --git a/components/elm/src/biogeochem/MaintenanceRespMod.F90 b/components/elm/src/biogeochem/MaintenanceRespMod.F90 index 54005275b29..829e5c742a0 100644 --- a/components/elm/src/biogeochem/MaintenanceRespMod.F90 +++ b/components/elm/src/biogeochem/MaintenanceRespMod.F90 @@ -70,7 +70,7 @@ subroutine readMaintenanceRespParams ( ncid ) call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun(msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) br_mr_Inst = tempr - + end subroutine readMaintenanceRespParams !----------------------------------------------------------------------- @@ -107,7 +107,7 @@ subroutine MaintenanceResp(bounds, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] patch vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) br_xr => veg_vp%br_xr , & ! Input: [real(r8) (:) ] base rate for excess respiration frac_veg_nosno => canopystate_vars%frac_veg_nosno_patch , & ! Input: [integer (:) ] fraction of vegetation not covered by snow (0 OR 1) [-] laisun => canopystate_vars%laisun_patch , & ! Input: [real(r8) (:) ] sunlit projected leaf area index @@ -160,7 +160,7 @@ subroutine MaintenanceResp(bounds, & ! calculate temperature corrections for each soil layer, for use in ! estimating fine root maintenance respiration with depth tcsoi(c,j) = Q10**((t_soisno(c,j)-SHR_CONST_TKFRZ - 20.0_r8)/10.0_r8) - + end do end do diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 index c1b2bfc5d65..f95b9a258cb 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 @@ -130,7 +130,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step @@ -147,7 +147,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp do j = 1, nlevdecomp do fc = 1,num_soilc c = filter_soilc(fc) - + ! N deposition and fixation (put all into NH4 pool) col_ns%smin_nh4_vr(c,j) = col_ns%smin_nh4_vr(c,j) + col_nf%ndep_to_sminn(c)*dt * ndep_prof(c,j) col_ns%smin_nh4_vr(c,j) = col_ns%smin_nh4_vr(c,j) + col_nf%nfix_to_sminn(c)*dt * nfixation_prof(c,j) @@ -219,7 +219,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp end do end if end do - + do j = 1, nlevdecomp ! column loop do fc = 1,num_soilc @@ -254,8 +254,8 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp end do ! end of column loop end do - - endif !end if is_active_betr_bgc + + endif !end if is_active_betr_bgc ! forest fertilization call get_curr_date(kyr, kmo, kda, mcsec) diff --git a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 index 87b9637d551..d80b844e600 100644 --- a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 +++ b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 @@ -397,7 +397,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, -1._r8, f_gresp_storage_to_xfer , s_gresp_storage,nelms) !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_carbon_d, nelms, f_gresp_storage_to_xfer) - + !initialize stoichiometric relationship between carbon production flux and corresponding state varaibles call spm_list_init(spm_list, 1._r8, f_cpool_to_leafc , s_leafc, nelms) call spm_list_insert(spm_list, 1._r8, f_leafc_xfer_to_leafc , s_leafc, nelms) @@ -432,7 +432,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, 1._r8, f_gresp_storage_to_xfer , s_gresp_xfer,nelms) call spm_list_insert(spm_list, 1._r8, f_cpool_to_gresp_storage , s_gresp_storage, nelms) - + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_carbon_p, nelms, f_gresp_storage_to_xfer) !initialize stoichiometry relationship between nutrient consumption and corresponding state variables @@ -474,7 +474,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, -1._r8, f_grainn_to_food , s_grainn, nelms) call spm_list_insert(spm_list, -1._r8, f_grainn_xfer_to_grainn , s_grainn_xfer,nelms) call spm_list_insert(spm_list, -1._r8, f_retransn_to_npool , s_retransn, nelms) - !turn the list into sparse matrix form + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_nutrient_d, nelms, f_supplement_to_plantn) !initialize stoichiometry relationship between nutrient production and corresponding state variables call spm_list_init(spm_list, 1._r8, f_retransn_to_npool , s_npool, nelms) @@ -512,7 +512,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, 1._r8, f_frootn_to_retransn , s_retransn, nelms) call spm_list_insert(spm_list, 1._r8, f_livestemn_to_retransn , s_retransn, nelms) call spm_list_insert(spm_list, 1._r8, f_livecrootn_to_retransn , s_retransn, nelms) - !turn the list into sparse matrix form + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_nutrient_p, nelms, f_supplement_to_plantn) end subroutine InitPhenoFluxLimiter !--------------------------------------------------------------------------- @@ -613,7 +613,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& real(r8) :: ar_p associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) harvdate => crop_vars%harvdate_patch & ! Input: [integer (:) ] harvest date ) ! set time steps @@ -847,7 +847,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& real(r8) :: dt associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) nf => veg_nf , & ns => veg_ns & ) @@ -1031,7 +1031,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) pf => veg_pf , & ps => veg_ps & ) diff --git a/components/elm/src/biogeochem/PhenologyMod.F90 b/components/elm/src/biogeochem/PhenologyMod.F90 index 08e9b94da52..c46e251b60c 100644 --- a/components/elm/src/biogeochem/PhenologyMod.F90 +++ b/components/elm/src/biogeochem/PhenologyMod.F90 @@ -163,7 +163,7 @@ subroutine readPhenolParams ( ncid ) allocate(PhenolParamsInst%lwtop ) ! ! read in parameters - ! + ! tString='crit_dayl' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun( msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) @@ -175,7 +175,7 @@ subroutine readPhenolParams ( ncid ) else tString='crit_dayl_stress' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) - if ( .not. readv ) then + if ( .not. readv ) then PhenolParamsInst%crit_dayl_stress = secspqtrday else PhenolParamsInst%crit_dayl_stress = tempr @@ -238,7 +238,7 @@ subroutine readPhenolParams ( ncid ) tString='lwtop_ann' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun( msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) - PhenolParamsInst%lwtop=tempr + PhenolParamsInst%lwtop=tempr !!!!========== Update to device ========= !!! !$acc update device(PhenolParamsInst%crit_dayl, & @@ -601,7 +601,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp, cnstate_vars) prev_dayl => grc_pp%prev_dayl , & ! Input: [real(r8) (:) ] daylength from previous time step (s) season_decid => veg_vp%season_decid , & ! Input: [real(r8) (:) ] binary flag for seasonal-deciduous leaf habit (0 or 1) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) t_soisno => col_es%t_soisno , & ! Input: [real(r8) (:,:) ] soil temperature (Kelvin) (-nlevsno+1:nlevgrnd) @@ -946,7 +946,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & leaf_long => veg_vp%leaf_long , & ! Input: [real(r8) (:) ] leaf longevity (yrs) froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) stress_decid => veg_vp%stress_decid , & ! Input: [real(r8) (:) ] binary flag for stress-deciduous leaf habit (0 or 1) soilpsi => soilstate_vars%soilpsi_col , & ! Input: [real(r8) (:,:) ] soil water potential in each soil layer (MPa) @@ -1445,7 +1445,7 @@ subroutine CropPhenology(num_pcropp, filter_pcropp, & froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) leafcn => veg_vp%leafcn , & ! Input: [real(r8) (:) ] leaf C:N (gC/gN) - manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) + manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) t_ref2m_min => veg_es%t_ref2m_min , & ! Input: [real(r8) (:) ] daily minimum of average 2 m height surface air temperature (K) t10 => veg_es%t_a10 , & ! Input: [real(r8) (:) ] 10-day running mean of the 2 m temperature (K) a5tmin => veg_es%t_a5min , & ! Input: [real(r8) (:) ] 5-day running mean of min 2-m temperature @@ -1988,7 +1988,7 @@ subroutine PerennialCropPhenology(num_ppercropp, filter_ppercropp, & froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) leafcn => veg_vp%leafcn , & ! Input: [real(r8) (:) ] leaf C:N (gC/gN) leafcp => veg_vp%leafcp , & ! Input: [real(r8) (:) ] leaf C:P (gC/gP) - manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) + manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) t10 => veg_es%t_a10 , & ! Input: [real(r8) (:) ] 10-day running mean of the 2 m temperature (K) a10tmin => veg_es%t_a10min , & ! Input: [real(r8) (:) ] 10-day running mean of min 2-m temperature fertnitro => crop_vars%fertnitro_patch , & ! Input: [real(r8) (:) ] max fertilizer to be applied in total (kgN/m2) @@ -2016,8 +2016,8 @@ subroutine PerennialCropPhenology(num_ppercropp, filter_ppercropp, & crop_seedc_to_leaf => veg_cf%crop_seedc_to_leaf , & ! Output: [real(r8) (:) ] (gC/m2/s) seed source to PFT-level - synthfert => veg_nf%synthfert , & ! Output: [real(r8) (:) ] (gN/m2/s) fertilizer applied each timestep - manure => veg_nf%manure , & ! Output: [real(r8) (:) ] (gN/m2/s) manure applied each timestep + synthfert => veg_nf%synthfert , & ! Output: [real(r8) (:) ] (gN/m2/s) fertilizer applied each timestep + manure => veg_nf%manure , & ! Output: [real(r8) (:) ] (gN/m2/s) manure applied each timestep fert_p => veg_pf%fert_p , & ! Output: [real(r8) (:) ] (gP/m2/s) phosphorus fertilizer applied each timestep fert_counter => veg_nf%fert_counter , & ! Output: [real(r8) (:) ] >0 fertilize; <=0 not (seconds) @@ -2550,22 +2550,22 @@ subroutine CropPlantDate (num_soilp, filter_soilp, num_pcropp, filter_pcropp, & xt(p,kmo) = xt(p,kmo) + t_ref2m(p) * fracday/ndaypm(kmo) ! monthly average temperature xp(p,kmo) = xp(p,kmo) + (forc_rain(t)+forc_snow(t))*dt ! monthly average precipitation - ! calculate the potential evapotranspiration + ! calculate the potential evapotranspiration netrad = fsa(p) + eflx_lwrad_net(p) ! moved this here because it is calculated too late call calculate_eto(t_ref2m(p), netrad, eflx_soil_grnd(p), forc_pbot(t), forc_rh(t), forc_wind(t), dt, ETout) ! monthly ETo ETo(p,kmo) = ETo(p,kmo) + ETout - + ! calculate the P:PET for each month - if ( abs(ETo(p,kmo)) > 0._r8) then + if ( abs(ETo(p,kmo)) > 0._r8) then p2ETo(p,kmo) = xp(p,kmo)/ETo(p,kmo) else ! P:PET is undefined. ! Setting to a fill value ( 'spval' ) would - ! require nested if statements due to + ! require nested if statements due to ! the weighting of previous years (i.e., p2ETo and prev_p2ETo_bar ) ! So, set to zero for simplicity. p2ETo(p,kmo) = 0._r8 - end if + end if if (nyrs_crop_active(p) == 0) then ! for the first year, use last years values prev_xt_bar(p,kmo) = xt(p,kmo) @@ -2617,7 +2617,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) onset_flag => cnstate_vars%onset_flag_patch , & ! Input: [real(r8) (:) ] onset flag onset_counter => cnstate_vars%onset_counter_patch , & ! Input: [real(r8) (:) ] onset days counter @@ -3277,7 +3277,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp) associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) livewdcn => veg_vp%livewdcn , & ! Input: [real(r8) (:) ] live wood (phloem and ray parenchyma) C:N (gC/gN) deadwdcn => veg_vp%deadwdcn , & ! Input: [real(r8) (:) ] dead wood (xylem and heartwood) C:N (gC/gN) diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 index 6e111116922..9297ab41e1b 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 @@ -75,7 +75,7 @@ subroutine PhosphorusStateUpdateDynPatch(bounds, num_soilc_with_inactive,& c = filter_soilc_with_inactive(fc) col_ps%prod10p(c) = col_ps%prod10p(c) + col_pf%dwt_prod10p_gain(c)*dt col_ps%prod100p(c) = col_ps%prod100p(c) + col_pf%dwt_prod100p_gain(c)*dt - col_ps%prod1p(c) = col_ps%prod1p(c) + col_pf%dwt_crop_productp_gain(c)*dt + col_ps%prod1p(c) = col_ps%prod1p(c) + col_pf%dwt_crop_productp_gain(c)*dt do j = 1,nlevdecomp @@ -125,7 +125,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step @@ -147,18 +147,18 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi do j = 1, nlevdecomp do fc = 1,num_soilc c = filter_soilc(fc) - + ! plant to litter fluxes ! phenology and dynamic landcover fluxes col_pf%decomp_ppools_sourcesink(c,j,i_met_lit) = & col_pf%phenology_p_to_litr_met_p(c,j) * dt - + col_pf%decomp_ppools_sourcesink(c,j,i_cel_lit) = & col_pf%phenology_p_to_litr_cel_p(c,j) * dt - + col_pf%decomp_ppools_sourcesink(c,j,i_lig_lit) = & col_pf%phenology_p_to_litr_lig_p(c,j) * dt - + end do end do end if diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 2f06ea90032..d077bea74b0 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -43,7 +43,7 @@ module VegetationPropertiesType real(r8), pointer :: dsladlai (:) => null() ! dSLA/dLAI, projected area basis [m^2/gC] real(r8), pointer :: leafcn (:) => null() ! leaf C:N (gC/gN) real(r8), pointer :: flnr (:) => null() ! fraction of leaf N in the Rubisco enzyme (gN Rubisco / gN leaf) - real(r8), pointer :: woody (:) => null() ! binary flag for woody lifeform (1=woody, 0=not woody) + real(r8), pointer :: woody (:) => null() woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) real(r8), pointer :: lflitcn (:) => null() ! leaf litter C:N (gC/gN) real(r8), pointer :: frootcn (:) => null() ! fine root C:N (gC/gN) real(r8), pointer :: livewdcn (:) => null() ! live wood (phloem and ray parenchyma) C:N (gC/gN) @@ -329,7 +329,7 @@ subroutine veg_vp_init(this) allocate( this%needleleaf(0:numpft)) ; this%needleleaf(:) =spval allocate( this%nfixer(0:numpft)) ; this%nfixer(:) =spval ! ----------------------------------------------------------------------------------------------------------- - ! NGEE Arctic show-vegetation interactions + ! NGEE Arctic snow-vegetation interactions allocate(this%bendresist(0:numpft)) ; this%bendresist(:) =spval allocate(this%vegshape(0:numpft)) ; this%vegshape(:) =spval allocate(this%stocking(0:numpft)) ; this%stocking(:) =spval From f5ee59a81d41bb2506a268e0bb54afed19505b6d Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Wed, 11 Sep 2024 15:41:27 -0400 Subject: [PATCH 058/366] Fix syntax error, missing comment marker --- components/elm/src/data_types/VegetationPropertiesType.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index d077bea74b0..d62167301cd 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -43,7 +43,7 @@ module VegetationPropertiesType real(r8), pointer :: dsladlai (:) => null() ! dSLA/dLAI, projected area basis [m^2/gC] real(r8), pointer :: leafcn (:) => null() ! leaf C:N (gC/gN) real(r8), pointer :: flnr (:) => null() ! fraction of leaf N in the Rubisco enzyme (gN Rubisco / gN leaf) - real(r8), pointer :: woody (:) => null() woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) + real(r8), pointer :: woody (:) => null() ! woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) real(r8), pointer :: lflitcn (:) => null() ! leaf litter C:N (gC/gN) real(r8), pointer :: frootcn (:) => null() ! fine root C:N (gC/gN) real(r8), pointer :: livewdcn (:) => null() ! live wood (phloem and ray parenchyma) C:N (gC/gN) From f30afb2f1db294c3ddda8e9dcb7bd9ffdba1a3fb Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 10 Sep 2024 19:04:55 +0000 Subject: [PATCH 059/366] switch linker back to F --- components/homme/cmake/HommeMacros.cmake | 8 +------- components/homme/cmake/machineFiles/chrysalis-bfb.cmake | 2 -- components/homme/cmake/machineFiles/chrysalis.cmake | 2 -- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 5610947cb29..6d073dbbe83 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -112,13 +112,7 @@ macro(createTestExec execName execType macroNP macroNC ADD_DEFINITIONS(-DHAVE_CONFIG_H) ADD_EXECUTABLE(${execName} ${EXEC_SOURCES}) - - if(SUNSPOT_MACHINE) - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX) - else() - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) - endif() - + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() diff --git a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake index fa1f1ac545c..b9f0d41a060 100644 --- a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake +++ b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake @@ -17,8 +17,6 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") -SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") - # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/chrysalis.cmake b/components/homme/cmake/machineFiles/chrysalis.cmake index 97bc682c954..68ff76ec808 100644 --- a/components/homme/cmake/machineFiles/chrysalis.cmake +++ b/components/homme/cmake/machineFiles/chrysalis.cmake @@ -17,8 +17,6 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") -SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") - # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") From a4c907fea9039e608d04799ce725de2a8f10eb1a Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 12 Sep 2024 18:21:11 -0500 Subject: [PATCH 060/366] more reviews --- components/data_comps/dlnd/src/dlnd_comp_mod.F90 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index 4d76c0fe1a1..fbd6e35cf87 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -105,10 +105,13 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & ! !DESCRIPTION: initialize dlnd model #ifdef HAVE_MOAB - use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + use iMOAB, only: iMOAB_DefineTagStorage, & iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & - iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo, iMOAB_WriteMesh + iMOAB_UpdateMeshInfo +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif #endif implicit none From 2c103c0e757ab099d82746e734dead767aa6742e Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 09:26:28 -0700 Subject: [PATCH 061/366] Allow interpinic to accept any netCDF filetype for output. Print warning and continue (instead of failing) when input has month dimension and output does not. --- .../elm/tools/interpinic/src/interpinic.F90 | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/components/elm/tools/interpinic/src/interpinic.F90 b/components/elm/tools/interpinic/src/interpinic.F90 index b75afd3a411..0b8af615577 100644 --- a/components/elm/tools/interpinic/src/interpinic.F90 +++ b/components/elm/tools/interpinic/src/interpinic.F90 @@ -146,9 +146,22 @@ subroutine interp_filei (fin, fout, cmdline) call check_ret (nf90_open(fin, NF90_NOWRITE, ncidi )) call check_ret (nf90_open(fout, NF90_NOWRITE, ncido )) call check_ret (nf_inq_format( ncido, ncformat )) - if ( ncformat /= NF_FORMAT_64BIT )then - write (6,*) 'error: output file is NOT in NetCDF large-file format!' - stop + ! 20240822 csz++ + ! Allow any format for output dataset + ! if ( ncformat /= NF_FORMAT_64BIT )then + ! write (6,*) 'error: output file is NOT in NetCDF large-file format!' + ! stop + ! 20240822 csz-- + if ( ncformat == NF_FORMAT_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_CLASSIC' + else if ( ncformat == NF_FORMAT_64BIT_OFFSET )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_OFFSET' + else if ( ncformat == NF_FORMAT_64BIT_DATA )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_DATA' + else if ( ncformat == NF_FORMAT_NETCDF4 )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4' + else if ( ncformat == NF_FORMAT_NETCDF4_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4_CLASSIC' end if call check_ret (nf90_inq_dimid(ncidi, "column", dimidcols )) @@ -214,13 +227,27 @@ subroutine interp_filei (fin, fout, cmdline) ret = nf90_inq_dimid(ncidi, "month", dimidmon) if (ret == NF90_NOERR) then call check_ret (nf90_inquire_dimension(ncidi, dimidmon, len=nlevmon)) - call check_ret (nf90_inq_dimid(ncido, "month", dimid )) - call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) - if (dimlen/=nlevmon) then - write (6,*) 'error: input and output nlevmon values disagree' - write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen - stop + ! 20240912 csz++ + ! Many restart files have "month" dimension in input dataset + ! It is only necessary that the output dataset contains "month" dimension + ! when a variable in the input dataset contains the "month" dimension + ! Otherwise, the "month" dimension will never be used + ! Warn rather than die when input has "month" and output does not + !call check_ret (nf90_inq_dimid(ncido, "month", dimid )) + ret = nf90_inq_dimid(ncido, "month", dimid ) + if ( ret == nf_ebaddim ) then + write (6,*) 'warning: input has "month" dimension and output does not' + write (6,*) 'warning: interpolation will fail if any input variable uses "month" dimension' + write (6,*) 'chill: many times the "month" dimension is superfluous so this might work...' + else + call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) + if (dimlen/=nlevmon) then + write (6,*) 'error: input and output nlevmon values disagree' + write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen + stop + end if end if + ! 20240912 csz-- else write (6,*) 'month dimension does NOT exist on the input dataset' dimidmon = -9999 @@ -321,7 +348,11 @@ subroutine interp_filei (fin, fout, cmdline) ! OK now, open the output file for writing ! call check_ret(nf90_close( ncido)) - call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) + ! 20240822 csz++ + ! Allow any format for output dataset + ! call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) + call check_ret (nf90_open(fout, NF90_WRITE, ncido )) + ! csz-- call addglobal (ncido, cmdline) @@ -1503,8 +1534,7 @@ subroutine addglobal (ncid, cmdline) character(len=10) :: time character(len= 5) :: zone character(len=18) :: datetime - character(len=256):: version = & - "$HeadURL: https://svn-ccsm-models.cgd.ucar.edu/clm2/trunk_tags/clm4_5_1_r085/models/lnd/clm/tools/clm4_5/interpinic/src/interpinic.F90 $" + character(len=256):: version = "" character(len=256) :: revision_id = "$Id: interpinic.F90 54953 2013-11-06 16:29:45Z sacks $" character(len=16) :: logname character(len=16) :: hostname From 4d1c5ea1a6af9249eb616ba408f6c4a0e36e293c Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 09:42:06 -0700 Subject: [PATCH 062/366] Place hexadecimal parameters within int() intrinsic in NaN module for modern compilers (per Xylar) --- .../tools/interpinic/src/shr_infnan_mod.F90 | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 index 638cad84d20..31ffb1dff32 100644 --- a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 +++ b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 @@ -2,11 +2,11 @@ module shr_infnan_mod -!! Inf_NaN_Detection module +!! Inf_NaN_Detection module !! Copyright(c) 2003, Lahey Computer Systems, Inc. -!! Copies of this source code, or standalone compiled files +!! Copies of this source code, or standalone compiled files !! derived from this source may not be sold without permission -!! from Lahey Computers Systems. All or part of this module may be +!! from Lahey Computers Systems. All or part of this module may be !! freely incorporated into executable programs which are offered !! for sale. Otherwise, distribution of all or part of this file is !! permitted, provided this copyright notice and header are included. @@ -22,12 +22,12 @@ module shr_infnan_mod !! isneginf(x) - test for a negative "infinite" value !! !! Each function accepts a single or double precision real argument, and -!! returns a true or false value to indicate the presence of the value +!! returns a true or false value to indicate the presence of the value !! being tested for. If the argument is array valued, the function returns !! a conformable logical array, suitable for use with the ANY function, or !! as a logical mask. !! -!! Each function operates by transferring the bit pattern from a real +!! Each function operates by transferring the bit pattern from a real !! variable to an integer container. Unless testing for + or - infinity, !! the sign bit is cleared to zero. The value is exclusive ORed with !! the value being tested for. The integer result of the IEOR function is @@ -48,14 +48,14 @@ module shr_infnan_mod integer, parameter :: Double = selected_int_kind(precision(1.0_r8)) ! Single precision IEEE values - integer(Single), parameter :: sNaN = Z"7FC00000" - integer(Single), parameter :: sPosInf = Z"7F800000" - integer(Single), parameter :: sNegInf = Z"FF800000" + integer(Single), parameter :: sNaN = int(Z"7FC00000") + integer(Single), parameter :: sPosInf = int(Z"7F800000") + integer(Single), parameter :: sNegInf = int(Z"FF800000") ! Double precision IEEE values - integer(Double), parameter :: dNaN = Z"7FF8000000000000" - integer(Double), parameter :: dPosInf = Z"7FF0000000000000" - integer(Double), parameter :: dNegInf = Z"FFF0000000000000" + integer(Double), parameter :: dNaN = int(Z"7FF8000000000000") + integer(Double), parameter :: dPosInf = int(Z"7FF0000000000000") + integer(Double), parameter :: dNegInf = int(Z"FFF0000000000000") ! Locatation of single and double precision sign bit (Intel) ! Subtract one because bit numbering starts at zero @@ -84,22 +84,22 @@ module shr_infnan_mod module procedure sisnan module procedure disnan #endif - end interface + end interface interface shr_infnan_isinf module procedure sisinf module procedure disinf - end interface - + end interface + interface shr_infnan_isposinf module procedure sisposinf module procedure disposinf - end interface - + end interface + interface shr_infnan_isneginf module procedure sisneginf module procedure disneginf - end interface + end interface integer :: shr_sisnan @@ -107,7 +107,7 @@ module shr_infnan_mod integer :: shr_disnan external :: shr_disnan -contains +contains ! ! If FORTRAN intrinsic's exist use them @@ -134,7 +134,7 @@ elemental function sisnan(x) result(res) res = isnan(x) #endif - end function + end function ! Double precision test for NaN elemental function disnan(d) result(res) @@ -156,7 +156,7 @@ elemental function disnan(d) result(res) res = isnan(d) #endif - end function + end function ! ! Otherwise link to a C function call that either uses the C90 isnan function or a x != x check @@ -176,13 +176,13 @@ function c_sisnan_1D(x) result(res) real(r4), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_sisnan(x(i)) /= 0) end do end function c_sisnan_1D - + function c_sisnan_2D(x) result(res) real(r4), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -195,7 +195,7 @@ function c_sisnan_2D(x) result(res) end do end do end function c_sisnan_2D - + function c_sisnan_3D(x) result(res) real(r4), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -210,7 +210,7 @@ function c_sisnan_3D(x) result(res) end do end do end function c_sisnan_3D - + function c_sisnan_4D(x) result(res) real(r4), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -227,7 +227,7 @@ function c_sisnan_4D(x) result(res) end do end do end function c_sisnan_4D - + function c_sisnan_5D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -246,7 +246,7 @@ function c_sisnan_5D(x) result(res) end do end do end function c_sisnan_5D - + function c_sisnan_6D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -267,7 +267,7 @@ function c_sisnan_6D(x) result(res) end do end do end function c_sisnan_6D - + function c_sisnan_7D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -290,7 +290,7 @@ function c_sisnan_7D(x) result(res) end do end do end function c_sisnan_7D - + function c_disnan_scalar(x) result(res) real(r8), intent(in) :: x logical :: res @@ -302,13 +302,13 @@ function c_disnan_1D(x) result(res) real(r8), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_disnan(x(i)) /= 0) end do end function c_disnan_1D - + function c_disnan_2D(x) result(res) real(r8), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -321,7 +321,7 @@ function c_disnan_2D(x) result(res) end do end do end function c_disnan_2D - + function c_disnan_3D(x) result(res) real(r8), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -336,7 +336,7 @@ function c_disnan_3D(x) result(res) end do end do end function c_disnan_3D - + function c_disnan_4D(x) result(res) real(r8), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -353,7 +353,7 @@ function c_disnan_4D(x) result(res) end do end do end function c_disnan_4D - + function c_disnan_5D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -372,7 +372,7 @@ function c_disnan_5D(x) result(res) end do end do end function c_disnan_5D - + function c_disnan_6D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -393,7 +393,7 @@ function c_disnan_6D(x) result(res) end do end do end function c_disnan_6D - + function c_disnan_7D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -418,48 +418,48 @@ function c_disnan_7D(x) result(res) end function c_disnan_7D #endif - + ! Single precision test for Inf elemental function sisinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(ibclr(transfer(x,sPosInf),SPSB), sPosInf) == 0 - end function + end function ! Double precision test for Inf elemental function disinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(ibclr(transfer(d,dPosInf),DPSB), dPosInf) == 0 - end function - + end function + ! Single precision test for +Inf elemental function sisposinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sPosInf), sPosInf) == 0 - end function + end function ! Double precision test for +Inf elemental function disposinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dPosInf), dPosInf) == 0 - end function - + end function + ! Single precision test for -Inf elemental function sisneginf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sNegInf), sNegInf) == 0 - end function + end function ! Double precision test for -Inf elemental function disneginf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dNegInf), dNegInf) == 0 - end function + end function end module shr_infnan_mod From bf6b5482bbb3dbb05ed859217d950a1907e65e14 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 13 Sep 2024 13:26:59 -0500 Subject: [PATCH 063/366] add MOAB root for improv machine at LCRC --- cime_config/machines/config_machines.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 30508f8a630..2308d3b9962 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3025,6 +3025,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6 /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH} $SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi} + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/improv/gnu; else echo "$MOAB_ROOT"; fi} ^lockedfile From 25fcf92996d401b7c674c8a7f18abad1c798583e Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 13 Sep 2024 14:21:12 -0700 Subject: [PATCH 064/366] add newline at end of file --- components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl index e3d4433d4a8..3a47a8bb2ff 100644 --- a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl +++ b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -31,4 +31,5 @@ seq_flux_mct_albdif = 0.07 seq_flux_mct_albdir = 0.07 seq_flux_atmocn_minwind = 1 -constant_zenith_deg = 42.04 \ No newline at end of file +constant_zenith_deg = 42.04 + From ef7aeca234bdd44b9bce518671e69489e13df6b6 Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 16:38:59 -0700 Subject: [PATCH 065/366] remove external qualifier from iargc definition --- components/elm/tools/interpinic/src/fmain.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/tools/interpinic/src/fmain.F90 b/components/elm/tools/interpinic/src/fmain.F90 index 0f55c12c67c..a67e6e2627f 100644 --- a/components/elm/tools/interpinic/src/fmain.F90 +++ b/components/elm/tools/interpinic/src/fmain.F90 @@ -14,7 +14,7 @@ program fmain character(len= 256) :: arg integer :: n !index integer :: nargs !number of arguments - integer, external :: iargc !number of arguments function + integer :: iargc !number of arguments function character(len=256) :: finidati !input initial dataset to read character(len=256) :: finidato !output initial dataset to create character(len=256) :: cmdline !input command line From ece56d136c5861288291243cae2f49e73414c250 Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 16:59:21 -0700 Subject: [PATCH 066/366] Remove commented-out code, CSZ markers, and dates last changed --- .../elm/tools/interpinic/src/interpinic.F90 | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/components/elm/tools/interpinic/src/interpinic.F90 b/components/elm/tools/interpinic/src/interpinic.F90 index 0b8af615577..3d890a14b89 100644 --- a/components/elm/tools/interpinic/src/interpinic.F90 +++ b/components/elm/tools/interpinic/src/interpinic.F90 @@ -146,12 +146,9 @@ subroutine interp_filei (fin, fout, cmdline) call check_ret (nf90_open(fin, NF90_NOWRITE, ncidi )) call check_ret (nf90_open(fout, NF90_NOWRITE, ncido )) call check_ret (nf_inq_format( ncido, ncformat )) - ! 20240822 csz++ + ! Allow any format for output dataset - ! if ( ncformat /= NF_FORMAT_64BIT )then - ! write (6,*) 'error: output file is NOT in NetCDF large-file format!' - ! stop - ! 20240822 csz-- + if ( ncformat == NF_FORMAT_CLASSIC )then write (6,*) 'info: output file is NF_FORMAT_CLASSIC' else if ( ncformat == NF_FORMAT_64BIT_OFFSET )then @@ -227,13 +224,13 @@ subroutine interp_filei (fin, fout, cmdline) ret = nf90_inq_dimid(ncidi, "month", dimidmon) if (ret == NF90_NOERR) then call check_ret (nf90_inquire_dimension(ncidi, dimidmon, len=nlevmon)) - ! 20240912 csz++ + ! Many restart files have "month" dimension in input dataset ! It is only necessary that the output dataset contains "month" dimension ! when a variable in the input dataset contains the "month" dimension ! Otherwise, the "month" dimension will never be used ! Warn rather than die when input has "month" and output does not - !call check_ret (nf90_inq_dimid(ncido, "month", dimid )) + ret = nf90_inq_dimid(ncido, "month", dimid ) if ( ret == nf_ebaddim ) then write (6,*) 'warning: input has "month" dimension and output does not' @@ -247,7 +244,6 @@ subroutine interp_filei (fin, fout, cmdline) stop end if end if - ! 20240912 csz-- else write (6,*) 'month dimension does NOT exist on the input dataset' dimidmon = -9999 @@ -348,11 +344,9 @@ subroutine interp_filei (fin, fout, cmdline) ! OK now, open the output file for writing ! call check_ret(nf90_close( ncido)) - ! 20240822 csz++ + ! Allow any format for output dataset - ! call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) call check_ret (nf90_open(fout, NF90_WRITE, ncido )) - ! csz-- call addglobal (ncido, cmdline) From dfeefffa109c1136068eefce1881121f67951a4f Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Mon, 16 Sep 2024 08:31:31 -0700 Subject: [PATCH 067/366] clean up new mock-walker SST section to make it more compact and less redundant --- .../data_comps/docn/src/docn_comp_mod.F90 | 49 +++---------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index ac127867365..43bac32bff7 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -1177,52 +1177,17 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) !------------------------------------------------------------------------------- ! RCEMIP phase 2 - Mock-Walker - - ! MW_295dT1p25 - mean SST = 295 K / dSST = 1.25 K - - if (sst_option == 11) then - mean_SST = 295 - TkFrz - delta_SST = 1.25 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_300dT0p625 - mean SST = 300 K / dSST = 0.625 K - if (sst_option == 12) then - mean_SST = 300 - TkFrz - delta_SST = 0.625 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_300dT1p25 - mean SST = 300 K / dSST = 1.25 K - if (sst_option == 13) then - mean_SST = 300 - TkFrz - delta_SST = 1.25 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_300dT2p5 - mean SST = 300 K / dSST = 2.5 K - if (sst_option == 14) then - mean_SST = 300 - TkFrz - delta_SST = 2.5 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_305dT1p25 - mean SST = 305 K / dSST = 1.25 K - if (sst_option == 15) then - mean_SST = 305 - TkFrz - delta_SST = 1.25 + if (sst_option>=11 .and. sst_option<=15) then + if (sst_option==11) then; mean_SST = 295 - TkFrz; delta_SST = 1.250; end if ! MW_295dT1p25 + if (sst_option==12) then; mean_SST = 300 - TkFrz; delta_SST = 0.625; end if ! MW_300dT0p625 + if (sst_option==13) then; mean_SST = 300 - TkFrz; delta_SST = 1.250; end if ! MW_300dT1p25 + if (sst_option==14) then; mean_SST = 300 - TkFrz; delta_SST = 2.500; end if ! MW_300dT2p5 + if (sst_option==15) then; mean_SST = 305 - TkFrz; delta_SST = 1.250; end if ! MW_305dT1p25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if + !------------------------------------------------------------------------------- end subroutine prescribed_sst From 3cb0e38377340333859ec0e1df3b5d4d4a0ca52d Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Mon, 16 Sep 2024 13:02:05 -0400 Subject: [PATCH 068/366] Moved tests to e3sm_land_exeshare test suite --- cime_config/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index d4c10f126b2..25558ef8ad8 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -49,6 +49,9 @@ "ERS.f19_g16.I20TRGSWCNPECACNTBC.elm-eca_f19_g16_I20TRGSWCNPECACNTBC", "ERS.f19_g16.I20TRGSWCNPRDCTCBC.elm-ctc_f19_g16_I20TRGSWCNPRDCTCBC", "ERS.r05_r05.ICNPRDCTCBC.elm-cbudget", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-snowveg_arctic", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", ) }, @@ -93,10 +96,7 @@ "SMS_Ly2_P1x1.1x1_smallvilleIA.IELMCNCROP.elm-fan", "SMS.r05_r05.IELM.elm-topounit", "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-snowveg_arctic", "ERS.r05_r05.IELM.elm-lnd_rof_2way", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", "ERS.r05_r05.IELM.elm-V2_ELM_MOSART_features", "ERS.ELM_USRDAT.IELM.elm-surface_water_dynamics" ) From a56bd4e2efdc2e14b975d2efe8272484be8f2a9e Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 16 Sep 2024 17:47:30 -0400 Subject: [PATCH 069/366] fix outdated path of output yaml files --- .../testdefs/testmods_dirs/eamxx/prod/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands index 435556401a2..a5ad8d6c668 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands @@ -61,7 +61,7 @@ else fi # set the output yaml files -output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/ -maxdepth 1 -type f) +output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/ -maxdepth 1 -type f) for file in ${output_yaml_files[@]}; do # if the word "coarse" is in the file name, do nothing if [[ "${file}" == *"_coarse.yaml" && "${hmapfile}" == "not-supported-yet" ]]; then From 6a6415a80c6bf1342c19abe789bfcac4dbf50d50 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 16 Sep 2024 19:45:04 -0400 Subject: [PATCH 070/366] edit logic to accomodate eamxx specs --- cime_config/config_archive.xml | 2 +- .../testdefs/testmods_dirs/eamxx/prod/shell_commands | 3 +++ ..._native.yaml => eamxx_output.decadal.1dailyAVG_native.yaml} | 2 +- ..._native.yaml => eamxx_output.decadal.1dailyMAX_native.yaml} | 2 +- ..._native.yaml => eamxx_output.decadal.1dailyMIN_native.yaml} | 2 +- ...INST_arm.yaml => eamxx_output.decadal.1hourlyINST_arm.yaml} | 2 +- ...ative.yaml => eamxx_output.decadal.1hourlyINST_native.yaml} | 2 +- ...coarse.yaml => eamxx_output.decadal.3hourlyAVG_coarse.yaml} | 2 +- ...oarse.yaml => eamxx_output.decadal.3hourlyINST_coarse.yaml} | 2 +- ...coarse.yaml => eamxx_output.decadal.6hourlyAVG_coarse.yaml} | 2 +- ...oarse.yaml => eamxx_output.decadal.6hourlyINST_coarse.yaml} | 2 +- ...ative.yaml => eamxx_output.decadal.6hourlyINST_native.yaml} | 2 +- ...G_coarse.yaml => eamxx_output.decadal.dailyAVG_coarse.yaml} | 2 +- 13 files changed, 15 insertions(+), 12 deletions(-) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1dailyAVG_native.yaml => eamxx_output.decadal.1dailyAVG_native.yaml} (81%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1dailyMAX_native.yaml => eamxx_output.decadal.1dailyMAX_native.yaml} (82%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1dailyMIN_native.yaml => eamxx_output.decadal.1dailyMIN_native.yaml} (80%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1hourlyINST_arm.yaml => eamxx_output.decadal.1hourlyINST_arm.yaml} (92%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1hourlyINST_native.yaml => eamxx_output.decadal.1hourlyINST_native.yaml} (85%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.3hourlyAVG_coarse.yaml => eamxx_output.decadal.3hourlyAVG_coarse.yaml} (96%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.3hourlyINST_coarse.yaml => eamxx_output.decadal.3hourlyINST_coarse.yaml} (96%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.6hourlyAVG_coarse.yaml => eamxx_output.decadal.6hourlyAVG_coarse.yaml} (94%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.6hourlyINST_coarse.yaml => eamxx_output.decadal.6hourlyINST_coarse.yaml} (91%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.6hourlyINST_native.yaml => eamxx_output.decadal.6hourlyINST_native.yaml} (90%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.dailyAVG_coarse.yaml => eamxx_output.decadal.dailyAVG_coarse.yaml} (97%) diff --git a/cime_config/config_archive.xml b/cime_config/config_archive.xml index f343fce81ba..8cb7a530538 100644 --- a/cime_config/config_archive.xml +++ b/cime_config/config_archive.xml @@ -29,7 +29,7 @@ r\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* rhist\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* - hi\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d*\.\d{4}-\d{2}-\d{2}-\d{5}\.nc$ + .*\.h\.(?!rhist\.).*\.nc$ diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands index a5ad8d6c668..2520f7b6b12 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands @@ -3,6 +3,7 @@ cime_root=$(./xmlquery --value CIMEROOT) input_data_dir=$(./xmlquery --value DIN_LOC_ROOT) atmchange=$cime_root/../components/eamxx/scripts/atmchange +case_name=$(./xmlquery --value CASE) # Change run length ./xmlchange RUN_STARTDATE="1994-10-01" @@ -82,6 +83,8 @@ for file in ${output_yaml_files[@]}; do sed -i "s|horiz_remap_file:.*_to_ne30.*|horiz_remap_file: ${hmapfile}|" ./$(basename ${file}) sed -i "s|horiz_remap_file:.*_to_DecadalSites.*|horiz_remap_file: ${armmapfile}|" ./$(basename ${file}) fi + # replace all filename prefixes so that st_archive works... + sed -i "s|eamxx_output.decadal|${case_name}.scream|" ./$(basename ${file}) done # TODO: diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml similarity index 81% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml index 4e1239b5c33..181f841eeb8 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyAVG_native.h +filename_prefix: eamxx_output.decadal.1dailyAVG_native.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml similarity index 82% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml index a7d10efe070..a8974c75b35 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMAX_native.h +filename_prefix: eamxx_output.decadal.1dailyMAX_native.h iotype: pnetcdf Averaging Type: Max Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml similarity index 80% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml index 653e194d278..8d48a1bedf6 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMIN_native.h +filename_prefix: eamxx_output.decadal.1dailyMIN_native.h iotype: pnetcdf Averaging Type: Min Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml index 5bb07048aed..52fc391ca4d 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_arm.h +filename_prefix: eamxx_output.decadal.1hourlyINST_arm.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml similarity index 85% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml index 7a221e89f1c..0aba4827ead 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_native.h +filename_prefix: eamxx_output.decadal.1hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml index 665294c6227..d429b11ebd1 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml index 42c64954508..a2faa1b971c 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml index 5e4aaed0738..437142ba559 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml similarity index 91% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml index e9e0f34d5e0..bb83718a8eb 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml similarity index 90% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml index bb7fd275abf..c69dc4b2212 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_native.h +filename_prefix: eamxx_output.decadal.6hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml similarity index 97% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml index 7c1990a7b56..2d1e6e7221e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.dailyAVG_coarse.h +filename_prefix: eamxx_output.decadal.dailyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 From 30ffb16650cfeeb54098495f630fd9698586d16c Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 17 Sep 2024 01:02:49 +0000 Subject: [PATCH 071/366] switch to 16,8 pair --- components/homme/src/share/cxx/ExecSpaceDefs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index c9ca8a0ecd9..4f3d97135fe 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -176,7 +176,7 @@ team_num_threads_vectors_for_gpu ( num_vectors ); } #else - return std::make_pair(4,16); + return std::make_pair(16,8); #endif } From 112fe79e60eeaa4d7ca0d85a54f24640b6a974f6 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 17 Sep 2024 01:03:44 +0000 Subject: [PATCH 072/366] adjust aurora flags --- components/homme/cmake/machineFiles/aurora-aot.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake index b6fe34a78d7..094ec888278 100644 --- a/components/homme/cmake/machineFiles/aurora-aot.cmake +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -43,9 +43,9 @@ SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") #AOT flags SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") -SET(SYCL_LINK_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fsycl-link-huge-device-code -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") -SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") From 0fcac3cc2906dee8a391f2d446bcf5286db38919 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 17 Sep 2024 01:04:25 +0000 Subject: [PATCH 073/366] adopt ekat sycl changes, TeamVectorRange --- .../share/cxx/utilities/scream_tridiag.hpp | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp index e18bbc4e7e2..26221db3955 100644 --- a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp +++ b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp @@ -128,6 +128,10 @@ int get_thread_id_within_team_gpu (const TeamMember& team) { // Can't use team.team_rank() here because vector direction also uses physical // threads but TeamMember types don't expose that information. return blockDim.x * threadIdx.y + threadIdx.x; +#elif defined(__SYCL_DEVICE_ONLY__) + auto item = team.item(); + return static_cast(item.get_local_range(1) * item.get_local_id(0) + + item.get_local_id(1)); #else assert(0); return -1; @@ -138,6 +142,9 @@ template KOKKOS_FORCEINLINE_FUNCTION int get_team_nthr_gpu (const TeamMember& team) { #if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ return blockDim.x * blockDim.y; +#elif defined __SYCL_DEVICE_ONLY__ + auto item = team.item(); + return static_cast(item.get_local_range(0) * item.get_local_range(1)); #else assert(0); return -1; @@ -161,6 +168,16 @@ KOKKOS_FORCEINLINE_FUNCTION int get_team_nthr (const Kokkos::Impl::HIPTeamMember& team) { return get_team_nthr_gpu(team); } #endif // KOKKOS_ENABLE_HIP + +#ifdef KOKKOS_ENABLE_SYCL +KOKKOS_FORCEINLINE_FUNCTION +int get_thread_id_within_team (const Kokkos::Impl::SYCLTeamMember& team) +{ return get_thread_id_within_team_gpu(team); } +KOKKOS_FORCEINLINE_FUNCTION +int get_team_nthr (const Kokkos::Impl::SYCLTeamMember& team) +{ return get_team_nthr_gpu(team); } +#endif // KOKKOS_ENABLE_SYCL + template KOKKOS_INLINE_FUNCTION const T& min (const T& a, const T& b) { return a < b ? a : b; } @@ -634,7 +651,7 @@ void bfb (const TeamMember& team, const auto f = [&] (const int& j) { impl::bfb_thomas_solve(dl, d, du, Kokkos::subview(X , Kokkos::ALL(), j)); }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nrhs), f); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nrhs), f); } template @@ -664,7 +681,7 @@ void bfb (const TeamMember& team, subview(du, ALL(), j), subview(X , ALL(), j)); }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nrhs), f); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nrhs), f); } } // namespace tridiag From 3e6959e0260e962bf402825108cb76286a32be6b Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 18 Sep 2024 07:17:31 -0700 Subject: [PATCH 074/366] Fixes a typo --- components/elm/docs/user-guide/interpinic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md index 63f8c30b03d..5127aeeb94a 100644 --- a/components/elm/docs/user-guide/interpinic.md +++ b/components/elm/docs/user-guide/interpinic.md @@ -15,7 +15,7 @@ The steps involved in creating a new IC files are as follows: The notes below provide an example of creating 1850 ELM IC file for the NARRM grid using E3SM v3 LR piControl from year = 0101. These notes are provided for Chrysalis. -### 1. Identification of the input ELM IC file +## 1. Identification of the input ELM IC file The identified input land condition file for this case is the following: @@ -39,7 +39,7 @@ elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c`date "+ ``` # Load relevant modules cd -eval $(./cime/CIME/Tools/get_case_env/get_case_env) +eval $(./cime/CIME/Tools/get_case_env) # change directory cd components/elm/tools/interpinic/src From ff40c9cff08029ec29b3a98f974f74357dab61aa Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 18 Sep 2024 07:32:45 -0700 Subject: [PATCH 075/366] Adds notes about using mksurfdata_map --- components/elm/docs/user-guide/index.md | 8 ++- .../elm/docs/user-guide/surface_dataset.md | 49 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 components/elm/docs/user-guide/surface_dataset.md diff --git a/components/elm/docs/user-guide/index.md b/components/elm/docs/user-guide/index.md index d33278ce761..bfab59d92b1 100644 --- a/components/elm/docs/user-guide/index.md +++ b/components/elm/docs/user-guide/index.md @@ -77,7 +77,11 @@ Using the above-mentioned settings: The [FATES User's Guide section on namelist options](https://fates-users-guide.readthedocs.io/en/latest/user/Namelist-Options-and-Run-Time-Modes.html) provides guidance on enabling these different FATES run modes. -## Generate land initial condition +## Create land surface dataset + +A new surface dataset for ELM is generated using `mksurfdata_map` and the notes about it are available [here](surface_dataset.md) -Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). +## Generate land initial condition + +Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). \ No newline at end of file diff --git a/components/elm/docs/user-guide/surface_dataset.md b/components/elm/docs/user-guide/surface_dataset.md new file mode 100644 index 00000000000..c513d2d1f59 --- /dev/null +++ b/components/elm/docs/user-guide/surface_dataset.md @@ -0,0 +1,49 @@ +# Creating an ELM surface dataset + +The notes describe the steps in creating an ELM surface dataset at 0.5x0.5 resolution for 1950 on Perlmutter. + +1. Load the appropriate modules. + +``` +cd +eval $(./cime/CIME/Tools/get_case_env) +``` + +2. Compile `mksurfdata_map`. + +``` +cd components/elm/tools/mksurfdata_map/src/ + +make clean +export USER_LDFLAGS="-L$NETCDF_DIR/lib -lnetcdf -lnetcdff -lnetcdf_intel" +export USER_LDFLAGS=$USER_LDFLAGS" -L$HDF5_DIR/lib -lhdf5 -lhdf5_fortran -lhdf5_hl_intel -lhdf5hl_fortran_intel" + +USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 +``` + +3. Build the namelist. This steps assumes that the resolution for which the new surface dataset is being created is a supported resolution. +If the surface dataset is being created for an unsupported resolution, 16 mapping files will have to be created to map the raw datasets +onto this unsupported resolution. + +``` +cd ../ + +RES=0.5x0.5 +YR=1950 +DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata + +./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT +mv namelist namelist.$RES.$YR +``` + +4. Run `mksurfdata_map` via an interactive job. + +``` +salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account e3sm + +RES=0.5x0.5 +YR=1950 +NAMELIST=namelist.$RES.$YR + +srun -n 1 ./mksurfdata_map < $NAMELIST +``` \ No newline at end of file From f61c6f8000a9418c7e5e9f2f6976d4bbb8352a89 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 18 Sep 2024 08:14:05 -0700 Subject: [PATCH 076/366] Few fixes --- components/elm/docs/user-guide/index.md | 3 +-- components/elm/docs/user-guide/interpinic.md | 12 +++++------ .../elm/docs/user-guide/surface_dataset.md | 20 ++++++++++--------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/components/elm/docs/user-guide/index.md b/components/elm/docs/user-guide/index.md index bfab59d92b1..43fe53d6655 100644 --- a/components/elm/docs/user-guide/index.md +++ b/components/elm/docs/user-guide/index.md @@ -81,7 +81,6 @@ provides guidance on enabling these different FATES run modes. A new surface dataset for ELM is generated using `mksurfdata_map` and the notes about it are available [here](surface_dataset.md) - ## Generate land initial condition -Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). \ No newline at end of file +Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md index 5127aeeb94a..fcb897834f2 100644 --- a/components/elm/docs/user-guide/interpinic.md +++ b/components/elm/docs/user-guide/interpinic.md @@ -19,7 +19,7 @@ The notes below provide an example of creating 1850 ELM IC file for the NARRM gr The identified input land condition file for this case is the following: -``` +```bash /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc ``` @@ -27,7 +27,7 @@ The identified input land condition file for this case is the following: Using an existing NARRM land IC and making a copy of it -``` +```bash cd components/elm/tools/interpinic cp /lcrc/group/e3sm/data/inputdata/lnd/clm2/initdata_map/elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1870-01-01-00000.c20240704.nc \ @@ -36,7 +36,7 @@ elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c`date "+ ## 3. Compiling `interpinic` -``` +```bash # Load relevant modules cd eval $(./cime/CIME/Tools/get_case_env) @@ -47,14 +47,13 @@ cd components/elm/tools/interpinic/src export USER_LDFLAGS="-L$NETCDF_C_DIR/lib -lnetcdf -L$NETCDF_F_DIR/lib -lnetcdff -L$HDF5_DIR/lib -lhdf5" USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 - ``` ## 4. Run `interpinic` -The `interpinic ` can then be run via the following batch job (e.g., `remap.r025_RRSwISC6to18E3r4.1850.batch`) to generate the initial condition. +The `interpinic` can then be run via the following batch job (e.g., `remap.r025_RRSwISC6to18E3r4.1850.batch`) to generate the initial condition. -``` +```bash >cat remap.r025_RRSwISC6to18E3r4.1850.batch #!/bin/sh @@ -76,4 +75,3 @@ srun -n 1 ./interpinic \ -i /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc \ -o elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c20240903.nc ``` - diff --git a/components/elm/docs/user-guide/surface_dataset.md b/components/elm/docs/user-guide/surface_dataset.md index c513d2d1f59..04d5b00f63c 100644 --- a/components/elm/docs/user-guide/surface_dataset.md +++ b/components/elm/docs/user-guide/surface_dataset.md @@ -2,16 +2,16 @@ The notes describe the steps in creating an ELM surface dataset at 0.5x0.5 resolution for 1950 on Perlmutter. -1. Load the appropriate modules. +## 1. Load the appropriate modules -``` +```bash cd eval $(./cime/CIME/Tools/get_case_env) ``` -2. Compile `mksurfdata_map`. +## 2. Compile `mksurfdata_map` -``` +```bash cd components/elm/tools/mksurfdata_map/src/ make clean @@ -21,11 +21,13 @@ export USER_LDFLAGS=$USER_LDFLAGS" -L$HDF5_DIR/lib -lhdf5 -lhdf5_fortran -lhdf5_ USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 ``` -3. Build the namelist. This steps assumes that the resolution for which the new surface dataset is being created is a supported resolution. +## Build the namelist + +This steps assumes that the resolution for which the new surface dataset is being created is a supported resolution. If the surface dataset is being created for an unsupported resolution, 16 mapping files will have to be created to map the raw datasets onto this unsupported resolution. -``` +```bash cd ../ RES=0.5x0.5 @@ -36,9 +38,9 @@ DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata mv namelist namelist.$RES.$YR ``` -4. Run `mksurfdata_map` via an interactive job. +## Run `mksurfdata_map` via an interactive job -``` +```bash salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account e3sm RES=0.5x0.5 @@ -46,4 +48,4 @@ YR=1950 NAMELIST=namelist.$RES.$YR srun -n 1 ./mksurfdata_map < $NAMELIST -``` \ No newline at end of file +``` From 644019ac981589be630435b115d93842ebe86489 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Wed, 18 Sep 2024 14:35:10 -0500 Subject: [PATCH 077/366] minor changes to improve clarity --- components/ww3/cime_config/buildlib_cmake | 25 ++++++++++------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/components/ww3/cime_config/buildlib_cmake b/components/ww3/cime_config/buildlib_cmake index 9df52bc35c6..42d92b2f0ad 100755 --- a/components/ww3/cime_config/buildlib_cmake +++ b/components/ww3/cime_config/buildlib_cmake @@ -45,21 +45,18 @@ def buildlib(bldroot, installpath, case): modeldir = "{}/WW3/model".format(repodir) builddir = "{}/wav".format(exeroot) - # TODO: these work dirs will have to be changed to live in the binary/build area. - # Achieving this will probably require a significant refactor of the ww3 infrastructure. - # Doing this stuff in-source not only clutters the repo, but also introduces potential race - # conditions if we were to try to build multiple ww3 cases simultaneously. - bindir1 = "{}/bin".format(modeldir) - bindir = "{}/wav/bin".format(exeroot) - shutil.copytree(bindir1, bindir) - auxdir1 = "{}/aux".format(modeldir) - auxdir = "{}/wav/aux".format(exeroot) - shutil.copytree(auxdir1, auxdir) - ftndir1 = "{}/ftn".format(modeldir) - ftndir = "{}/wav/ftn".format(exeroot) - shutil.copytree(ftndir1, ftndir) + # work dirs are placed in the binary/build area. + bindir_source = "{}/bin".format(modeldir) + bindir = "{}/bin".format(builddir) + shutil.copytree(bindir_source, bindir) + auxdir_source = "{}/aux".format(modeldir) + auxdir = "{}/aux".format(builddir) + shutil.copytree(auxdir_source, auxdir) + ftndir_source = "{}/ftn".format(modeldir) + ftndir = "{}/ftn".format(builddir) + shutil.copytree(ftndir_source, ftndir) - tmpdir = "{}/wav/tmp".format(exeroot) + tmpdir = "{}/tmp".format(builddir) # Run w3_setup to create wwatch3.env file env_file = os.path.join(bindir, "wwatch3.env") From 03158e3e8fddfb860ea1818acea9986c81746f7a Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Wed, 18 Sep 2024 16:16:13 -0500 Subject: [PATCH 078/366] Updating CRYO1850* compsets to v3 ELM settings --- cime_config/allactive/config_compsets.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index e66f1e473d8..b17bfd7c7ca 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -383,12 +383,12 @@ CRYO1850 - 1850SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYO1850-4xCO2 - 1850SOI_EAM%CMIP6-4xCO2_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6-4xCO2_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV @@ -398,7 +398,7 @@ CRYO1850-DISMF - 1850SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV From 6fbecc13ce1906f9d3162b1dedeb295c80769d1e Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 18 Sep 2024 17:20:50 -0500 Subject: [PATCH 079/366] Add support for new JRA forcing through year 2023 --- .../data_comps/datm/cime_config/config_component.xml | 10 +++++++--- .../data_comps/drof/cime_config/config_component.xml | 7 ++++++- components/mpas-ocean/cime_config/config_compsets.xml | 5 +++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/components/data_comps/datm/cime_config/config_component.xml b/components/data_comps/datm/cime_config/config_component.xml index 9145bdf4b16..5b884ecd1ad 100644 --- a/components/data_comps/datm/cime_config/config_component.xml +++ b/components/data_comps/datm/cime_config/config_component.xml @@ -10,7 +10,7 @@ This file may have atm desc entries. --> - Data driven ATM + Data driven ATM QIAN data set QIAN with water isotopes CRUNCEP data set @@ -28,8 +28,9 @@ COREv2 normal year forcing COREv2 interannual forcing interannual JRA55 forcing - interannual JRA55 forcing, v1.5, through 2020 interannual JRA55 forcing, v1.4, through 2018 + interannual JRA55 forcing, v1.5, through 2020 + interannual JRA55 forcing, v1.5, through 2023 JRA55 Repeat Year Forcing v1.3 1984-1985 JRA55 Repeat Year Forcing v1.3 1990-1991 JRA55 Repeat Year Forcing v1.3 2003-2004 @@ -63,8 +64,9 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). CORE2_NYF CORE2_IAF CORE_IAF_JRA - IAF_JRA_1p5 CORE_IAF_JRA_1p4_2018 + IAF_JRA_1p5 + IAF_JRA_1p5 CORE_RYF8485_JRA CORE_RYF9091_JRA CORE_RYF0304_JRA @@ -391,6 +393,7 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). $DATM_CLMNCEP_YR_START 1 1 + 1 $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START @@ -488,6 +491,7 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). 2003 2016 2020 + 2023 1979 1979 diff --git a/components/data_comps/drof/cime_config/config_component.xml b/components/data_comps/drof/cime_config/config_component.xml index b5e0a8071fa..aede006caf2 100644 --- a/components/data_comps/drof/cime_config/config_component.xml +++ b/components/data_comps/drof/cime_config/config_component.xml @@ -13,7 +13,7 @@ --> - Data runoff model + Data runoff model NULL mode COREv2 normal year forcing: COREv2 normal year forcing: @@ -24,6 +24,7 @@ COREv2 interannual year forcing: COREv2 interannual year forcing: CPLHIST mode: + JRA55 interannual forcing, v1.5, through 2023 JRA55 interannual forcing, v1.5, through 2020 JRA55 interannual forcing, v1.5, through 2020, no rofi or rofl around AIS JRA55 interannual forcing, v1.4, through 2018 @@ -61,6 +62,7 @@ DIATREN_IAF_AIS55_RX1 CPLHIST IAF_JRA + IAF_JRA_1p5 IAF_JRA_1p5 IAF_JRA_1p5_AIS0ROF IAF_JRA_1p4_2018 @@ -165,6 +167,7 @@ 1 1 1 + 1 run_component_drof env_run.xml @@ -179,6 +182,7 @@ 1958 1958 1958 + 1958 run_component_drof env_run.xml @@ -193,6 +197,7 @@ 2016 2018 2020 + 2023 run_component_drof env_run.xml diff --git a/components/mpas-ocean/cime_config/config_compsets.xml b/components/mpas-ocean/cime_config/config_compsets.xml index be36f010347..52ad20a5ab7 100644 --- a/components/mpas-ocean/cime_config/config_compsets.xml +++ b/components/mpas-ocean/cime_config/config_compsets.xml @@ -67,6 +67,11 @@ 2000_DATM%JRA-1p5_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p5_SGLC_SWAV + + GMPAS-JRA1p5-2023 + 2000_DATM%JRA-1p5-2023_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p5-2023_SGLC_SWAV + + GMPAS-OECO-JRA1p4 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%OECODATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV From a8763c20197cee06943a527eba000a6e0b93fb34 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 19 Sep 2024 08:58:52 -0700 Subject: [PATCH 080/366] Cleanup to fix issues from earlier PR causing sporadic test failures --- .../bld/namelist_files/namelist_defaults_mali.xml | 2 +- components/mpas-albany-landice/cime_config/buildnml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml index 0450eb44f11..8dc305533ca 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml @@ -18,7 +18,7 @@ 'fo' -'none' +'fo' 3 0.25 .false. diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 9489b6dfa8f..3b38ed39c43 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -280,7 +280,9 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + if mali_use_albany: + lines.append(' ') + lines.append('') lines.append('') lines.append(' - -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240501.nc - - - @@ -2056,76 +2049,83 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/northamericax4v1pg2/map_0.5x0.5_GSDTG2000_to_northamericax4v1pg2_nomask_aave_da_c210112.nc - + -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_AVHRR_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_MODIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.9x1.25_GRDC_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_10x10min_IGBPmergeICESatGIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_10x10min_nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_360x720cru_cruncep_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_GLOBE-Gardner_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_LandScan2004_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_MODIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_USGS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_IGBP-GSDP_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_ISRIC-WISE_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_GSDTG2000_to_0.125x0.125_nomask_aave_da_c190725.nc - + -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_AVHRR_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.9x1.25_GRDC_to_0.25x0.25_nomask_aave_da_c240124.nc lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_IGBPmergeICESatGIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.25x0.25_nomask_aave_da_c240123.nc lnd/clm2/mappingdata/maps/0.25x0.25/map_360x720cru_cruncep_to_0.25x0.25_nomask_aave_da_c240124.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_LandScan2004_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_USGS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_IGBP-GSDP_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_ISRIC-WISE_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_360x720_cruncep_to_0.25x0.25_nomask_aave_da_c240124.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_GSDTG2000_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240501.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.1x0.1_nomask_to_0.25x0.25_nomask_aave_da_c240308.nc + lnd/clm2/mappingdata/maps/antarcticax4v1/map_0.5x0.5_AVHRR_to_antarcticax4v1_nomask_aave_da_c210130.nc diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 7063d7a1105..d36bf5565ce 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -1502,7 +1502,7 @@ If TRUE, irrigation will be active (find surface datasets with active irrigation +"1000,850,1100,1350,1600,1850,1855,1865,1875,1885,1895,1905,1915,1925,1935,1945,1950,1955,1965,1975,1980,1985,1995,2000,2005,2010,2015,2025,2035,2045,2055,2065,2075,2085,2095,2100,2105"> Year to simulate and to provide datasets for (such as surface datasets, initial conditions, aerosol-deposition, Nitrogen deposition rates etc.) A sim_year of 1000 corresponds to data used for testing only, NOT corresponding to any real datasets. A sim_year greater than 2005 corresponds to rcp scenario data From 1204d21fdfe4c3af43ddce44ce4ac945de7e458d Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Fri, 20 Sep 2024 11:04:58 -0700 Subject: [PATCH 088/366] Adds notes on creating surface dataset with 10 glacier layers --- .../elm/docs/user-guide/surface_dataset.md | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/components/elm/docs/user-guide/surface_dataset.md b/components/elm/docs/user-guide/surface_dataset.md index 04d5b00f63c..5adaad100d6 100644 --- a/components/elm/docs/user-guide/surface_dataset.md +++ b/components/elm/docs/user-guide/surface_dataset.md @@ -23,9 +23,9 @@ USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --included ## Build the namelist -This steps assumes that the resolution for which the new surface dataset is being created is a supported resolution. +This step assumes that the resolution for which the new surface dataset is being created is a supported resolution. If the surface dataset is being created for an unsupported resolution, 16 mapping files will have to be created to map the raw datasets -onto this unsupported resolution. +onto this unsupported resolution. The `namelist` file with default number of glaciers (equal to zero) can be generated as: ```bash cd ../ @@ -35,7 +35,16 @@ YR=1950 DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata ./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT -mv namelist namelist.$RES.$YR +``` + +An example of generating the namelist for 0.25 deg (`r025`) resolution for 1980 with 10 glacier layers is as follows: + +```bash +RES=r025 +YR=1980 +DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata + +./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT -glc_nec 10 ``` ## Run `mksurfdata_map` via an interactive job @@ -43,9 +52,5 @@ mv namelist namelist.$RES.$YR ```bash salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account e3sm -RES=0.5x0.5 -YR=1950 -NAMELIST=namelist.$RES.$YR - -srun -n 1 ./mksurfdata_map < $NAMELIST +srun -n 1 ./mksurfdata_map < namelist ``` From 227f736b3b82d51b8d750514a977bf2c72cfe34e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 19 Sep 2024 20:00:32 -0700 Subject: [PATCH 089/366] Updates lc machines Signed-off-by: Jason Boutte --- .../machines/cmake_macros/intel_dane.cmake | 6 ++ .../machines/cmake_macros/intel_ruby.cmake | 6 ++ cime_config/machines/config_batch.xml | 36 +++------ cime_config/machines/config_machines.xml | 76 ++++++++++++------- 4 files changed, 71 insertions(+), 53 deletions(-) diff --git a/cime_config/machines/cmake_macros/intel_dane.cmake b/cime_config/machines/cmake_macros/intel_dane.cmake index 8091325c6ce..ef25a97b300 100644 --- a/cime_config/machines/cmake_macros/intel_dane.cmake +++ b/cime_config/machines/cmake_macros/intel_dane.cmake @@ -1,4 +1,10 @@ string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") + +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck/lib") + set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/cmake_macros/intel_ruby.cmake b/cime_config/machines/cmake_macros/intel_ruby.cmake index 8091325c6ce..e874bfb7eaf 100644 --- a/cime_config/machines/cmake_macros/intel_ruby.cmake +++ b/cime_config/machines/cmake_macros/intel_ruby.cmake @@ -1,4 +1,10 @@ string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") + +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5/lib") + set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 13cfdfb337b..923e497367f 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -237,33 +237,17 @@ - - squeue - sbatch - scancel - #SBATCH - (\d+)$ - --dependency=afterok:jobid - --dependency=afterany:jobid - : - %H:%M:%S - --mail-user - --mail-type - none, all, begin, end, fail - - --export=ALL - -p {{ job_queue }} - -J {{ job_id }} - -N {{ num_nodes }} - -n {{ total_tasks }} - -t {{ job_wallclock_time }} - -o {{ job_id }}.out - -e {{ job_id }}.err - -A {{ project }} - + + + pbatch + pdebug + + + + - pbatch - pdebug + pbatch + pdebug diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index ff0c583fc93..249cfdfc5a8 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3065,9 +3065,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 56 56 @@ -3076,8 +3076,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -3089,24 +3097,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion + cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 - cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core + mvapich2/2.3.7-ll7cmqm + hdf5/1.10.7-ewjpbjd + netcdf-c/4.4.1.1-vaxofek + netcdf-fortran/4.4.4-3pzbx2u + parallel-netcdf/1.11.0-tzgdala $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2 + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5 @@ -3121,9 +3132,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 224 112 @@ -3132,8 +3143,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -3145,24 +3164,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core + mvapich2/2.3.7-27jao34 + hdf5/1.10.7-766kapa + netcdf-c/4.4.1.1-2uznnlw + netcdf-fortran/4.4.4-itpstyo + parallel-netcdf/1.11.0-26sxm4m $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck From 7d2259d531e2c089e013b48d29ea0a1f01a3fae8 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 23 Sep 2024 14:02:13 -0700 Subject: [PATCH 090/366] update fates default parameter file with new ngee-a shrubs --- components/elm/bld/namelist_files/namelist_defaults.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index beeae785859..7d2390d267b 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -134,7 +134,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.36.0.0_12pft_c240517.nc +lnd/clm2/paramdata/fates_params_api.36.1.0_12pft_c240926.nc lnd/clm2/paramdata/CNP_parameters_c131108.nc From af5d1e33c7fa0073e36677ae4d5b41ea1a8fe0ec Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Tue, 24 Sep 2024 12:35:51 -0500 Subject: [PATCH 091/366] Updates to bgc registry and history fields -Corrects units/descriptions of bgc fields in registry -Defines column integrated and cell average bgc diagnostics -Updates streams build to include new diagnostics for bgc runs BFB --- components/mpas-seaice/cime_config/buildnml | 57 +- components/mpas-seaice/src/Registry.xml | 758 +++++++++++++----- .../src/shared/mpas_seaice_icepack.F | 424 +++++++++- 3 files changed, 1015 insertions(+), 224 deletions(-) diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index c38ca08a31d..d43fbdc151c 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -924,22 +924,41 @@ def buildnml(case, caseroot, compname): lines.append(' ') if ice_bgc == 'ice_bgc': - lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -999,15 +1018,35 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') diff --git a/components/mpas-seaice/src/Registry.xml b/components/mpas-seaice/src/Registry.xml index 7c842ca2f3c..78830161c62 100644 --- a/components/mpas-seaice/src/Registry.xml +++ b/components/mpas-seaice/src/Registry.xml @@ -866,7 +866,7 @@ icepack_name="tr_bgc_PON" /> @@ -948,77 +948,77 @@ possible_values="positive real number" icepack_name="frazil_scav" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1473,32 +1474,32 @@ possible_values="-1 = entirely in the mobile phase; 0 = retention dominated; 1 = release dominated; 0.5 = equal but rapid exchange; 2 = equal but slow exchange" icepack_name="zaerotype_dust4" /> - - - - - - - - - - - - - - - - - - - - + + @@ -2701,17 +2705,17 @@ - - - - - - - - - - - + + + + + + + + + + + @@ -2885,11 +2889,13 @@ @@ -3422,241 +3474,325 @@ + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -5887,6 +6242,7 @@ diff --git a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F index 36899b4b10d..fa3b28c92c8 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F @@ -3923,7 +3923,11 @@ subroutine column_biogeochemistry(domain) abortFlag = icepack_warnings_aborted() call get_cice_tracer_array_category(block, ciceTracerObject, & - tracerArrayCategory, iCell, setGetPhysicsTracers, setGetBGCTracers) + tracerArrayCategory, iCell, setGetPhysicsTracers, setGetBGCTracers) + + if (config_use_vertical_tracers) & + call define_total_biogeochemistry_array_cell(block, ciceTracerObject, totalVerticalBiologyIce(:,iCell), & + totalVerticalBiologySnow(:,iCell), iCell) if (checkCarbon) then call seaice_total_carbon_content_category(block,totalCarbonCatFinal(:,iCell),iceAreaCategory(1,:,:),iceVolumeCategory(1,:,:),iCell) @@ -8648,10 +8652,13 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace nzAerosols type(MPAS_pool_type), pointer :: & - tracers_aggregate + tracers_aggregate, & + biogeochemistry real(kind=RKIND), dimension(:), pointer :: & - brineFractionCell + brineFractionCell, & + carbonToNitrogenRatioAlgae, & + carbonToNitrogenRatioDON real(kind=RKIND), dimension(:,:), pointer :: & skeletalAlgaeConcCell, & @@ -8699,14 +8706,21 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalParticulateIronIceCell, & verticalDissolvedIronIceCell, & verticalAerosolsIceCell, & - verticalAerosolsSnowCell + verticalAerosolsSnowCell, & + verticalDOCLabileIceCell, & + verticalAlgaeTotalCarbonIceCell, & + verticalBCTotalIceCell, & + verticalDustTotalIceCell, & + verticalBCTotalSnowCell, & + verticalDustTotalSnowCell integer :: & iBioTracers, & iBioCount, & iLayers, & iIceCount, & - iSnowCount + iSnowCount, & + nAeroType call MPAS_pool_get_config(block % configs, "config_use_skeletal_biochemistry", config_use_skeletal_biochemistry) call MPAS_pool_get_config(block % configs, "config_use_vertical_biochemistry", config_use_vertical_biochemistry) @@ -8736,6 +8750,7 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace call MPAS_pool_get_dimension(block % dimensions, "nDissolvedIron", nDissolvedIron) call MPAS_pool_get_subpool(block % structs, "tracers_aggregate", tracers_aggregate) + call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistry) call MPAS_pool_get_array(tracers_aggregate, "skeletalAlgaeConcCell", skeletalAlgaeConcCell) call MPAS_pool_get_array(tracers_aggregate, "skeletalDOCConcCell", skeletalDOCConcCell) @@ -8784,6 +8799,14 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace call MPAS_pool_get_array(tracers_aggregate, "verticalAerosolsSnowCell", verticalAerosolsSnowCell) call MPAS_pool_get_array(tracers_aggregate, "verticalSalinityCell", verticalSalinityCell) call MPAS_pool_get_array(tracers_aggregate, "brineFractionCell", brineFractionCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalAlgaeTotalCarbonIceCell", verticalAlgaeTotalCarbonIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDOCLabileIceCell", verticalDOCLabileIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalBCTotalIceCell", verticalBCTotalIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDustTotalIceCell", verticalDustTotalIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDustTotalSnowCell", verticalDustTotalSnowCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalBCTotalSnowCell", verticalBCTotalSnowCell) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioAlgae", carbonToNitrogenRatioAlgae) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioDON", carbonToNitrogenRatioDON) ! biogeochemistry ! brine height fraction @@ -8866,12 +8889,15 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace ! algal nitrogen do iBioTracers = 1, nAlgae iIceCount = (iBioTracers-1)*nBioLayersP1 + verticalAlgaeTotalCarbonIceCell(:,iCell) = 0.0_RKIND do iLayers = 1,nBioLayersP1 iBiocount = iBiocount + 1 verticalAlgaeConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_algaeConc(iBioTracers)+iLayers-1) verticalAlgaeIceCell(iLayers+iIceCount,iCell) = verticalAlgaeConcCell(iBioCount,iCell) + verticalAlgaeTotalCarbonIceCell(iLayers,iCell) = verticalAlgaeTotalCarbonIceCell(iLayers,iCell) + & + carbonToNitrogenRatioAlgae(iBioTracers) * verticalAlgaeConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBiocount = iBiocount + 1 @@ -8894,6 +8920,7 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace if (config_use_carbon) then iBioCount = 0 + verticalDOCLabileIceCell(:,iCell) = 0.0_RKIND ! DOC do iBioTracers = 1, nDOC @@ -8904,6 +8931,8 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalDOCConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_DOCConc(iBioTracers) + iLayers-1) verticalDOCIceCell(iLayers+iIceCount,iCell) = verticalDOCConcCell(iBioCount,iCell) + verticalDOCLabileIceCell(iLayers, iCell) = verticalDOCLabileIceCell(iLayers, iCell) + & + verticalDOCConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -8942,6 +8971,8 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalDONConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_DONConc(iBioTracers) + iLayers-1) verticalDONIceCell(iLayers+iIceCount,iCell) = verticalDONConcCell(iBioCount,iCell) + verticalDOCLabileIceCell(iLayers, iCell) = verticalDOCLabileIceCell(iLayers, iCell) + & + verticalDONConcCell(iBioCount,iCell) * carbonToNitrogenRatioDON(iBioTracers) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -9058,15 +9089,25 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace ! black carbon and dust aerosols if (config_use_zaerosols) then iBioCount = 0 + verticalDustTotalIceCell(:,iCell) = 0.0_RKIND + verticalBCTotalIceCell(:,iCell) = 0.0_RKIND + verticalDustTotalSnowCell(:,iCell) = 0.0_RKIND + verticalBCTotalSnowCell(:,iCell) = 0.0_RKIND + do iBioTracers = 1, nzAerosols iIceCount = (iBioTracers-1)*nBioLayersP1 iSnowCount = (iBioTracers-1)*2 - + nAeroType = 1 + if (iBioTracers .gt. 2) nAeroType = 0 do iLayers = 1,nBioLayersP1 iBioCount = iBioCount + 1 verticalAerosolsConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_verticalAerosolsConc(iBioTracers)+iLayers-1) verticalAerosolsIceCell(iLayers+iIceCount,iCell) = verticalAerosolsConcCell(iBioCount,iCell) + verticalBCTotalIceCell(iLayers, iCell) = verticalBCTotalIceCell(iLayers, iCell) + nAeroType * & + verticalAerosolsConcCell(iBioCount,iCell) + verticalDustTotalIceCell(iLayers, iCell) = verticalDustTotalIceCell(iLayers, iCell) + (1-nAeroType) * & + verticalAerosolsConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -9074,6 +9115,10 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace tracerArrayCell(tracerObject % index_verticalAerosolsConc(iBioTracers)+iLayers-1) verticalAerosolsSnowCell(iLayers-nBioLayersP1+iSnowCount,iCell) = & verticalAerosolsConcCell(iBioCount,iCell) + verticalBCTotalSnowCell(iLayers-nBioLayersP1, iCell) = verticalBCTotalSnowCell(iLayers-nBioLayersP1, iCell) + & + nAeroType * verticalAerosolsConcCell(iBioCount,iCell) + verticalDustTotalSnowCell(iLayers-nBioLayersP1,iCell) = verticalDustTotalSnowCell(iLayers-nBioLayersP1,iCell) + & + (1-nAeroType) * verticalAerosolsConcCell(iBioCount,iCell) enddo enddo endif @@ -9088,6 +9133,274 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace end subroutine get_cice_biogeochemistry_tracer_array_cell +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! define_total_biogeochemistry_array_cell +! +!> \brief +!> \author Nicole Jeffery, LANL +!> \date 16rd September 2024 +!> \details +!> +! Defines total ice and snow arrays for biogeochemistry and aerosols diagnostics +! +!----------------------------------------------------------------------- + + subroutine define_total_biogeochemistry_array_cell(block, tracerObject, totalVerticalBioIceCell, & + totalVerticalBioSnowCell, iCell) + + type(block_type), intent(in) :: & + block + + type(ciceTracerObjectType), intent(in) :: & + tracerObject + + real(kind=RKIND), dimension(:), intent(in):: & + totalVerticalBioIceCell, & + totalVerticalBioSnowCell + + integer, intent(in) :: & + iCell + + logical, pointer :: & + config_use_vertical_biochemistry, & + config_use_nitrate, & + config_use_carbon, & + config_use_ammonium, & + config_use_silicate, & + config_use_DMS, & + config_use_nonreactive, & + config_use_humics, & + config_use_DON, & + config_use_iron, & + config_use_zaerosols + + integer, pointer :: & + nAlgae, & + nDOC, & + nDIC, & + nDON, & + nParticulateIron, & + nDissolvedIron, & + nzAerosols + + type(MPAS_pool_type), pointer :: & + biogeochemistry + + real(kind=RKIND), dimension(:), pointer :: & + totalVerticalDiatomIce, & + totalVerticalSmallPlanktonIce, & + totalVerticalPhaeocystisIce, & + totalVerticalNitrateIce, & + totalVerticalPolysaccharidsIce, & + totalVerticalLipidsIce, & + totalVerticalDICIce, & + totalVerticalAmmoniumIce, & + totalVerticalSilicateIce, & + totalVerticalDMSPpIce, & + totalVerticalDMSPdIce, & + totalVerticalDMSIce, & + totalVerticalNonreactiveIce, & + totalVerticalHumicsIce, & + totalVerticalProteinsIce, & + totalVerticalDissolvedIronIce, & + totalVerticalParticulateIronIce, & + totalVerticalDustIce, & + totalVerticalDustSnow, & + totalVerticalBC1Ice, & + totalVerticalBC1Snow, & + totalVerticalBC2Ice, & + totalVerticalBC2Snow, & + totalVerticalBCIce, & + totalVerticalBCSnow, & + totalVerticalDissolvedIronSnow, & + totalVerticalAlgaeCarbonIce, & + totalVerticalDOCLabileIce, & + carbonToNitrogenRatioAlgae, & + carbonToNitrogenRatioDON + + call MPAS_pool_get_config(block % configs, "config_use_vertical_biochemistry", config_use_vertical_biochemistry) + call MPAS_pool_get_config(block % configs, "config_use_nitrate", config_use_nitrate) + call MPAS_pool_get_config(block % configs, "config_use_carbon", config_use_carbon) + call MPAS_pool_get_config(block % configs, "config_use_ammonium",config_use_ammonium) + call MPAS_pool_get_config(block % configs, "config_use_silicate",config_use_silicate) + call MPAS_pool_get_config(block % configs, "config_use_DMS",config_use_DMS) + call MPAS_pool_get_config(block % configs, "config_use_nonreactive",config_use_nonreactive) + call MPAS_pool_get_config(block % configs, "config_use_humics",config_use_humics) + call MPAS_pool_get_config(block % configs, "config_use_DON",config_use_DON) + call MPAS_pool_get_config(block % configs, "config_use_iron",config_use_iron) + call MPAS_pool_get_config(block % configs, "config_use_zaerosols",config_use_zaerosols) + + call MPAS_pool_get_dimension(block % dimensions, "nzAerosols", nzAerosols) + call MPAS_pool_get_dimension(block % dimensions, "nAlgae", nAlgae) + call MPAS_pool_get_dimension(block % dimensions, "nDOC", nDOC) + call MPAS_pool_get_dimension(block % dimensions, "nDIC", nDIC) + call MPAS_pool_get_dimension(block % dimensions, "nDON", nDON) + call MPAS_pool_get_dimension(block % dimensions, "nParticulateIron", nParticulateIron) + call MPAS_pool_get_dimension(block % dimensions, "nDissolvedIron", nDissolvedIron) + + call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistry) + + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDiatomIce", totalVerticalDiatomIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalSmallPlanktonIce", totalVerticalSmallPlanktonIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalPhaeocystisIce", totalVerticalPhaeocystisIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalPolysaccharidsIce", totalVerticalPolysaccharidsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalLipidsIce", totalVerticalLipidsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDICIce", totalVerticalDICIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalProteinsIce", totalVerticalProteinsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalNitrateIce", totalVerticalNitrateIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalSilicateIce", totalVerticalSilicateIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalAmmoniumIce", totalVerticalAmmoniumIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSIce", totalVerticalDMSIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSPpIce", totalVerticalDMSPpIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSPdIce", totalVerticalDMSPdIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalNonreactiveIce", totalVerticalNonreactiveIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalHumicsIce", totalVerticalHumicsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalParticulateIronIce", totalVerticalParticulateIronIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDissolvedIronIce", totalVerticalDissolvedIronIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDissolvedIronSnow", totalVerticalDissolvedIronSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDustIce", totalVerticalDustIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDustSnow", totalVerticalDustSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC1Ice", totalVerticalBC1Ice) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC1Snow", totalVerticalBC1Snow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC2Ice", totalVerticalBC2Ice) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC2Snow", totalVerticalBC2Snow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBCIce", totalVerticalBCIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBCSnow", totalVerticalBCSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioAlgae", carbonToNitrogenRatioAlgae) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioDON", carbonToNitrogenRatioDON) + + ! biogeochemistry + + if (config_use_vertical_biochemistry) then + + ! algal nitrogen + totalVerticalDiatomIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(1)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalDiatomIce(iCell) * carbonToNitrogenRatioAlgae(1) + + SELECT CASE (nAlgae) + CASE (2) + totalVerticalSmallPlanktonIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(2)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalAlgaeCarbonIce(iCell) + & + totalVerticalSmallPlanktonIce(iCell) * carbonToNitrogenRatioAlgae(2) + CASE (3) + totalVerticalSmallPlanktonIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(2)) + totalVerticalPhaeocystisIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(3)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalAlgaeCarbonIce(iCell) + & + totalVerticalSmallPlanktonIce(iCell) * carbonToNitrogenRatioAlgae(2) + & + totalVerticalPhaeocystisIce(iCell) * carbonToNitrogenRatioAlgae(3) + END SELECT + end if + + ! nitrate + if (config_use_nitrate) & + totalVerticalNitrateIce(iCell) = totalVerticalBioIceCell(tracerObject % index_nitrateConcLayer) + + if (config_use_carbon) then + + ! DOC + totalVerticalDOCLabileIce(iCell) = 0.0_RKIND + SELECT CASE (nDOC) + CASE (1) + totalVerticalPolysaccharidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalPolysaccharidsIce(iCell) + CASE (2) + totalVerticalPolysaccharidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalLipidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalPolysaccharidsIce(iCell) + totalVerticalLipidsIce(iCell) + END SELECT + + ! DIC + SELECT CASE (nDIC) + CASE (1) + totalVerticalDICIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DICConcLayer(1)) + END SELECT + end if + + if (config_use_DON) then + + ! DON + SELECT CASE (nDON) + CASE (1) + totalVerticalProteinsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DONConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalDOCLabileIce(iCell) + & + totalVerticalProteinsIce(iCell)* carbonToNitrogenRatioDON(1) + END SELECT + end if + + ! ammonium + if (config_use_ammonium) & + totalVerticalAmmoniumIce(iCell) = totalVerticalBioIceCell(tracerObject % index_ammoniumConcLayer) + + ! silicate + if (config_use_silicate) & + totalVerticalSilicateIce(iCell) = totalVerticalBioIceCell(tracerObject % index_silicateConcLayer) + + ! DMS, DMSPp, DMSPd + if (config_use_DMS) then + totalVerticalDMSIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSConcLayer) + totalVerticalDMSPpIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSPpConcLayer) + totalVerticalDMSPdIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSPdConcLayer) + end if + + ! nonreactive + if (config_use_nonreactive) & + totalVerticalNonreactiveIce(iCell) = totalVerticalBioIceCell(tracerObject % index_nonreactiveConcLayer) + + ! humic material (refractory DOC) + if (config_use_humics) then + totalVerticalHumicsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_humicsConcLayer) + end if + + if (config_use_iron) then + + ! ParticulateIron + SELECT CASE (nParticulateIron) + CASE (1) + totalVerticalParticulateIronIce(iCell) = totalVerticalBioIceCell(tracerObject % index_particulateIronConcLayer(1)) + END SELECT + + ! DissolvedIron + SELECT CASE (nDissolvedIron) + CASE (1) + totalVerticalDissolvedIronIce(iCell) = totalVerticalBioIceCell(tracerObject % index_dissolvedIronConcLayer(1)) + totalVerticalDissolvedIronSnow(iCell) = totalVerticalBioSnowCell(tracerObject % index_dissolvedIronConcLayer(1)) + END SELECT + + end if + + ! black carbon and dust aerosols + if (config_use_zaerosols) then + + SELECT CASE (nzAerosols) + CASE (1) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + CASE (2) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC2Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBC2Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBC2Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + totalVerticalBC2Snow(iCell) + CASE (3) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC2Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBC2Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalDustIce(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(3)) + totalVerticalDustSnow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(3)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBC2Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + totalVerticalBC2Snow(iCell) + END SELECT + end if + + end subroutine define_total_biogeochemistry_array_cell + !----------------------------------------------------------------------- ! Init CICE parameters !----------------------------------------------------------------------- @@ -14699,7 +15012,35 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) ! zSalinityFlux, & !echmod deprecate ! zSalinityGDFlux, & !echmod deprecate totalChlorophyll, & - totalCarbonContentCell + totalCarbonContentCell, & + totalVerticalDiatomIce, & + totalVerticalSmallPlanktonIce, & + totalVerticalPhaeocystisIce, & + totalVerticalNitrateIce, & + totalVerticalPolysaccharidsIce, & + totalVerticalLipidsIce, & + totalVerticalDICIce, & + totalVerticalAmmoniumIce, & + totalVerticalSilicateIce, & + totalVerticalDMSPpIce, & + totalVerticalDMSPdIce, & + totalVerticalDMSIce, & + totalVerticalNonreactiveIce, & + totalVerticalHumicsIce, & + totalVerticalProteinsIce, & + totalVerticalDissolvedIronIce, & + totalVerticalParticulateIronIce, & + totalVerticalDustIce, & + totalVerticalDustSnow, & + totalVerticalBC1Ice, & + totalVerticalBC1Snow, & + totalVerticalBC2Ice, & + totalVerticalBC2Snow, & + totalVerticalBCIce, & + totalVerticalBCSnow, & + totalVerticalAlgaeCarbonIce, & + totalVerticalDOCLabileIce, & + totalVerticalDissolvedIronSnow real(kind=RKIND), dimension(:,:), pointer :: & oceanBioFluxes, & @@ -14747,13 +15088,12 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridSalinityIceCell", bgridSalinityIceCell) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridPorosityIceCell", bgridPorosityIceCell) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridTemperatureIceCell", bgridTemperatureIceCell) - - primaryProduction = 0.0_RKIND - totalChlorophyll = 0.0_RKIND - netSpecificAlgalGrowthRate = 0.0_RKIND - bgridSalinityIceCell = 0.0_RKIND - bgridPorosityIceCell = 0.0_RKIND - bgridTemperatureIceCell = 0.0_RKIND + primaryProduction = 0.0_RKIND + totalChlorophyll = 0.0_RKIND + netSpecificAlgalGrowthRate = 0.0_RKIND + bgridSalinityIceCell = 0.0_RKIND + bgridPorosityIceCell = 0.0_RKIND + bgridTemperatureIceCell = 0.0_RKIND end if @@ -14764,6 +15104,34 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBiologyIce", totalVerticalBiologyIce) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBiologySnow", totalVerticalBiologySnow) call MPAS_pool_get_array(biogeochemistryPool, "totalCarbonContentCell", totalCarbonContentCell) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDiatomIce", totalVerticalDiatomIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalSmallPlanktonIce", totalVerticalSmallPlanktonIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalPhaeocystisIce", totalVerticalPhaeocystisIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalPolysaccharidsIce", totalVerticalPolysaccharidsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalLipidsIce", totalVerticalLipidsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDICIce", totalVerticalDICIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalProteinsIce", totalVerticalProteinsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalNitrateIce", totalVerticalNitrateIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalSilicateIce", totalVerticalSilicateIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAmmoniumIce", totalVerticalAmmoniumIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSIce", totalVerticalDMSIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSPpIce", totalVerticalDMSPpIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSPdIce", totalVerticalDMSPdIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalNonreactiveIce", totalVerticalNonreactiveIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalHumicsIce", totalVerticalHumicsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalParticulateIronIce", totalVerticalParticulateIronIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDissolvedIronIce", totalVerticalDissolvedIronIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDissolvedIronSnow", totalVerticalDissolvedIronSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDustIce", totalVerticalDustIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDustSnow", totalVerticalDustSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC1Ice", totalVerticalBC1Ice) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC1Snow", totalVerticalBC1Snow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC2Ice", totalVerticalBC2Ice) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC2Snow", totalVerticalBC2Snow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCIce", totalVerticalBCIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCSnow", totalVerticalBCSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) netBrineHeight = 0.0_RKIND @@ -14773,6 +15141,34 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBiologyIce = 0.0_RKIND totalVerticalBiologySnow = 0.0_RKIND totalCarbonContentCell = 0.0_RKIND + totalVerticalDiatomIce = 0.0_RKIND + totalVerticalSmallPlanktonIce = 0.0_RKIND + totalVerticalPhaeocystisIce = 0.0_RKIND + totalVerticalPolysaccharidsIce = 0.0_RKIND + totalVerticalLipidsIce = 0.0_RKIND + totalVerticalDICIce = 0.0_RKIND + totalVerticalProteinsIce = 0.0_RKIND + totalVerticalNitrateIce = 0.0_RKIND + totalVerticalSilicateIce = 0.0_RKIND + totalVerticalAmmoniumIce = 0.0_RKIND + totalVerticalDMSIce = 0.0_RKIND + totalVerticalDMSPpIce = 0.0_RKIND + totalVerticalDMSPdIce = 0.0_RKIND + totalVerticalNonreactiveIce = 0.0_RKIND + totalVerticalHumicsIce = 0.0_RKIND + totalVerticalParticulateIronIce = 0.0_RKIND + totalVerticalDissolvedIronIce = 0.0_RKIND + totalVerticalDissolvedIronSnow = 0.0_RKIND + totalVerticalBC1Ice = 0.0_RKIND + totalVerticalBC2Ice = 0.0_RKIND + totalVerticalDustIce = 0.0_RKIND + totalVerticalBC1Snow = 0.0_RKIND + totalVerticalBC2Snow = 0.0_RKIND + totalVerticalBCSnow = 0.0_RKIND + totalVerticalBCSnow = 0.0_RKIND + totalVerticalDustSnow = 0.0_RKIND + totalVerticalAlgaeCarbonIce = 0.0_RKIND + totalVerticalDOCLabileIce = 0.0_RKIND endif From 2413f42432a1e635117335c31d3bc7feef4f285d Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 24 Sep 2024 14:10:12 -0500 Subject: [PATCH 092/366] Hommexx: Temporarily work around some EAMxx-Hommexx incompatibilities. These are due to PR #6594 and break the E3SM-repo EAMxx build. Once the SCREAM and E3SM repos are unified, we can back out these workarounds. The workarounds are isolated to components/homme, keeping the commit separation of components/eamxx (SCREAM repo) and components/homme (E3SM repo) clean. --- .../homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp | 1 + components/homme/src/share/cxx/HommexxEnums.hpp | 6 ++++++ components/homme/src/share/cxx/SimulationParams.hpp | 1 + .../src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp | 1 + components/homme/src/theta-l_kokkos/prim_driver_mod.F90 | 6 ++++-- 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index b433a48c2ab..aa21f5c16f2 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -91,6 +91,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = disable_diagnostics; params.use_moisture = use_moisture; + params.moisture = params.use_moisture ? MoistDry::MOIST : MoistDry::DRY; //todo-repo-unification params.use_cpstar = use_cpstar; params.transport_alg = transport_alg; // SphereOperators parameters; preqx supports only the sphere. diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 06abbf35adb..f41ed9d3651 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -40,6 +40,12 @@ enum class ComparisonOp { // =================== Run parameters enums ====================== // +//todo-repo-unification Remove this enum in favor of bool +// SimulationParams::use_moisture once we change EAMxx to use +// use_moisture. Search "todo-repo-unification" for other bits of code to +// remove. +enum class MoistDry { MOIST, DRY }; + enum class ForcingAlg : int { FORCING_OFF =-1, FORCING_0 = 0, diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index 4f36962b16c..9247f54a352 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -24,6 +24,7 @@ struct SimulationParams TimeStepType time_step_type; bool use_moisture; + MoistDry moisture; //todo-repo-unification RemapAlg remap_alg; TestCase test_case; ForcingAlg ftype = ForcingAlg::FORCING_OFF; diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 40c4ae64dc9..e96b838f096 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -114,6 +114,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = (bool)disable_diagnostics; params.use_moisture = (bool)use_moisture; + params.moisture = params.use_moisture ? MoistDry::MOIST : MoistDry::DRY; //todo-repo-unification params.use_cpstar = (bool)use_cpstar; params.transport_alg = transport_alg; params.theta_hydrostatic_mode = (bool)theta_hydrostatic_mode; diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index eae8544ca86..e7cc245a2bd 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -353,12 +353,14 @@ subroutine prim_init_elements_views (elem) end subroutine prim_init_elements_views subroutine prim_init_kokkos_functors (allocate_buffer) - use iso_c_binding, only : c_int + !todo-repo-unification Remove the use of c_bool here. It's used in purely + ! F90 code. + use iso_c_binding, only : c_int, c_bool use theta_f2c_mod, only : init_functors_c, init_boundary_exchanges_c ! ! Optional Input ! - logical, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + logical(kind=c_bool), intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally integer(kind=c_int) :: ab ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, From b5c3c7f64d2334d5cf5015eb2c9c0e5d0e0a7a0f Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 23 Sep 2024 19:25:02 -0500 Subject: [PATCH 093/366] fix sat hist capability and add test Ressurecting a long-forgotten capability to output variable along ship/flight tracks, and adding a test to (weakly try to) ensure it doesn't break again. The code is rewritten such that it uses a different IO logic (more serial) and support is added for different types of output variables. Fixes #6543 [BFB] Note: I don't have full provenance from where the code edits are from. I recall attempting to fix this capability in July and August 2024, and I recall trying different versions of available CAM code to do so. But the final version isn't easily traceable (so it could be a mix of edits from CAM and debug edits to make it work in current EAM). For reference, here's the CAM code: https://github.com/ESCOMP/CAM/blob/cam_cesm2_1_rel/src/control/sat_hist.F90. I don't claim ownership of the code at all, and I am happy to give "authorship" to whomever claims it. --- cime_config/tests.py | 1 + .../testmods_dirs/eam/sathist_F2010/readme | 1 + .../eam/sathist_F2010/shell_commands | 3 + .../eam/sathist_F2010/user_nl_eam | 7 + components/eam/src/control/sat_hist.F90 | 261 ++++++++++++------ 5 files changed, 181 insertions(+), 92 deletions(-) create mode 100644 components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme create mode 100644 components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands create mode 100644 components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b8397..c3d7dc75740 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -207,6 +207,7 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", + "SMS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme new file mode 100644 index 00000000000..35f64eb4b9a --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme @@ -0,0 +1 @@ +test for sat hist capability components/eam/src/control/sat_hist.F90 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands new file mode 100644 index 00000000000..0dc0f9cc72c --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands @@ -0,0 +1,3 @@ +#!/bin/bash + +./xmlchange RUN_STARTDATE=2015-01-01 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam new file mode 100644 index 00000000000..146e2da835c --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam @@ -0,0 +1,7 @@ +&satellite_options_nl + sathist_mfilt = 10000, + sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/sathist_master_19700410-20150419_c20150616.nc' + sathist_hfilename_spec = '%c.sathist_master_19700410-20150419_c20150616.%y-%m-%d-%s.nc' + sathist_nclosest = 1 + sathist_ntimestep = 1 + sathist_fincl = 'T', 'PS' diff --git a/components/eam/src/control/sat_hist.F90 b/components/eam/src/control/sat_hist.F90 index 64b978ac0f1..af8e330e557 100644 --- a/components/eam/src/control/sat_hist.F90 +++ b/components/eam/src/control/sat_hist.F90 @@ -6,18 +6,20 @@ module sat_hist use perf_mod, only: t_startf, t_stopf - use shr_kind_mod, only: r8 => shr_kind_r8 + use shr_kind_mod, only: r4 => shr_kind_r4 + use shr_kind_mod, only: r8 => shr_kind_r8, cl=>shr_kind_cl use cam_logfile, only: iulog - use ppgrid, only: pcols, pver, begchunk, endchunk + use ppgrid, only: pcols, pver, pverp, begchunk, endchunk use cam_history_support, only: fieldname_lenp2, max_string_len, ptapes use spmd_utils, only: masterproc, iam use cam_abortutils, only: endrun - use pio, only: file_desc_t, iosystem_desc_t, iosystem_desc_t, var_desc_t, io_desc_t - use pio, only: pio_openfile, pio_redef, pio_enddef, pio_inq_dimid, pio_inq_varid, pio_seterrorhandling, pio_def_var + use pio, only: file_desc_t,iosystem_desc_t, var_desc_t, io_desc_t + use pio, only: pio_inq_dimid, pio_inq_varid + use pio, only: pio_seterrorhandling, pio_def_var use pio, only: pio_inq_dimlen, pio_get_att, pio_put_att, pio_get_var, pio_put_var, pio_write_darray - use pio, only: pio_real, pio_int, pio_double - use pio, only: PIO_WRITE,PIO_NOWRITE, PIO_NOERR, PIO_BCAST_ERROR, PIO_INTERNAL_ERROR, PIO_Rearr_box, PIO_GLOBAL + use pio, only: pio_real,pio_double + use pio, only: PIO_NOWRITE, PIO_NOERR, PIO_BCAST_ERROR, PIO_INTERNAL_ERROR, PIO_GLOBAL use spmd_utils, only: mpicom #ifdef SPMD use mpishorthand, only: mpichar, mpiint @@ -82,6 +84,7 @@ module sat_hist real(r8), parameter :: rad2deg = 180._r8/pi ! degrees per radian + contains !------------------------------------------------------------------------------- @@ -189,14 +192,13 @@ end subroutine sat_hist_readnl subroutine sat_hist_init use cam_pio_utils, only: cam_pio_openfile use ioFileMod, only: getfil - use spmd_utils, only: npes use time_manager, only: get_step_size use string_utils, only: to_lower, GLC implicit none character(len=max_string_len) :: locfn ! Local filename - integer :: ierr, dimid, i + integer :: ierr, dimid character(len=128) :: date_format @@ -406,17 +408,15 @@ end subroutine sat_hist_define !------------------------------------------------------------------------------- subroutine sat_hist_write( tape , nflds, nfils) - use ppgrid, only : pcols, begchunk, endchunk use phys_grid, only: phys_decomp use dyn_grid, only: dyn_decomp use cam_history_support, only : active_entry - use pio, only : pio_file_is_open - implicit none + use pio, only : pio_file_is_open, pio_syncfile type(active_entry) :: tape integer, intent(in) :: nflds integer, intent(inout) :: nfils - integer :: t, f, i, ncols, nocols + integer :: ncols, nocols integer :: ierr integer, allocatable :: col_ndxs(:) @@ -430,9 +430,13 @@ subroutine sat_hist_write( tape , nflds, nfils) real(r8),allocatable :: phs_dists(:) integer :: coldim - - integer :: io_type - logical :: has_dyn_flds + logical :: has_dyn_flds = .false. + logical :: has_phys_srf_flds = .false. + logical :: has_phys_lev_flds = .false. + logical :: has_phys_ilev_flds = .false. + logical :: has_dyn_srf_flds = .false. + logical :: has_dyn_lev_flds = .false. + logical :: has_dyn_ilev_flds = .false. if (.not.has_sat_hist) return @@ -456,13 +460,11 @@ subroutine sat_hist_write( tape , nflds, nfils) allocate( mlons(nocols) ) allocate( phs_dists(nocols) ) - has_dyn_flds = .false. - dyn_flds_loop: do f=1,nflds - if ( tape%hlist(f)%field%decomp_type == dyn_decomp ) then - has_dyn_flds = .true. - exit dyn_flds_loop - endif - enddo dyn_flds_loop + call scan_flds( tape, nflds & + , has_phys_srf_flds, has_phys_lev_flds, has_phys_ilev_flds & + , has_dyn_srf_flds, has_dyn_lev_flds, has_dyn_ilev_flds ) + + has_dyn_flds = has_dyn_srf_flds .or. has_dyn_lev_flds .or. has_dyn_ilev_flds call get_indices( obs_lats, obs_lons, ncols, nocols, has_dyn_flds, col_ndxs, chk_ndxs, & fdyn_ndxs, ldyn_ndxs, phs_owners, dyn_owners, mlats, mlons, phs_dists ) @@ -479,16 +481,35 @@ subroutine sat_hist_write( tape , nflds, nfils) call write_record_coord( tape, mlats(:), mlons(:), phs_dists(:), ncols, nfils ) - do f=1,nflds + ! dump columns of 2D fields + if (has_phys_srf_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, 1, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_srf_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, 1, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif - select case (tape%hlist(f)%field%decomp_type) - case (phys_decomp) - call dump_columns(tape%File, tape%hlist(f), nocols, nfils, col_ndxs(:), chk_ndxs(:), phs_owners(:) ) - case (dyn_decomp) - call dump_columns(tape%File, tape%hlist(f), nocols, nfils, fdyn_ndxs(:), ldyn_ndxs(:), dyn_owners(:) ) - end select + ! dump columns of 3D fields defined on mid pres levels + if (has_phys_lev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pver, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_lev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pver, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif - enddo + ! dump columns of 3D fields defined on interface pres levels + if (has_phys_ilev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pverp, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_ilev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pverp, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif deallocate( col_ndxs, chk_ndxs, fdyn_ndxs, ldyn_ndxs, phs_owners, dyn_owners ) deallocate( mlons, mlats, phs_dists ) @@ -501,92 +522,150 @@ subroutine sat_hist_write( tape , nflds, nfils) end subroutine sat_hist_write !------------------------------------------------------------------------------- - subroutine dump_columns( File, hitem, ncols, nfils, fdims, ldims, owners ) - use cam_history_support, only: field_info, hentry, hist_coords, fillvalue - use pio, only: pio_initdecomp, pio_freedecomp, pio_setframe, pio_iam_iotask, pio_setdebuglevel, pio_offset_kind + subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, owners, decomp ) + use cam_history_support, only: field_info, hentry, fillvalue + use pio, only: pio_setframe, pio_offset_kind + use spmd_utils, only: mpi_real4, mpi_real8, mpicom, mpi_sum type(File_desc_t),intent(inout) :: File - type(hentry), intent(in), target :: hitem + type(hentry), intent(in), target :: hitems(:) + integer, intent(in) :: nflds integer, intent(in) :: ncols + integer, intent(in) :: nlevs integer, intent(in) :: nfils integer, intent(in) :: fdims(:) integer, intent(in) :: ldims(:) integer, intent(in) :: owners(:) + integer, intent(in) :: decomp + type(field_info), pointer :: field type(var_desc_t) :: vardesc - type(iosystem_desc_t), pointer :: sat_iosystem - type(io_desc_t) :: iodesc - integer :: t, ierr, ndims - integer, allocatable :: dimlens(:) + integer :: ierr - real(r8), allocatable :: buf(:) - integer, allocatable :: dof(:) - integer :: i,k, cnt + real(r8) :: sbuf1d(ncols),rbuf1d(ncols) + real(r4) :: buf1d(ncols) + real(r8) :: sbuf2d(nlevs,ncols), rbuf2d(nlevs,ncols) + real(r4) :: buf2d(nlevs,ncols) + integer :: i,k,f, cnt call t_startf ('sat_hist::dump_columns') - sat_iosystem => File%iosystem - field => hitem%field - vardesc = hitem%varid(1) - - - ndims=1 - if(associated(field%mdims)) then - ndims = size(field%mdims)+1 - else if(field%numlev>1) then - ndims=2 - end if - allocate(dimlens(ndims)) - dimlens(ndims)=ncols - if(ndims>2) then - do i=1,ndims-1 - dimlens(i)=hist_coords(field%mdims(i))%dimsize - enddo - else if(field%numlev>1) then - dimlens(1) = field%numlev - end if - - - allocate( buf( product(dimlens) ) ) - allocate( dof( product(dimlens) ) ) + do f = 1,nflds + field => hitems(f)%field + + if (field%numlev==nlevs .and. field%decomp_type==decomp) then + vardesc = hitems(f)%varid(1) + + if (nlevs==1) then + sbuf1d = 0.0_r8 + rbuf1d = 0.0_r8 + do i=1,ncols + if ( iam == owners(i) ) then + sbuf1d(i) = hitems(f)%hbuf( fdims(i), 1, ldims(i) ) + endif + enddo + call mpi_allreduce(sbuf1d,rbuf1d,ncols,mpi_real8, mpi_sum, mpicom, ierr) + buf1d(:) = real(rbuf1d(:),r4) + ierr = pio_put_var(File, vardesc, (/nfils/),(/ncols/), buf1d(:)) + else + sbuf2d = 0.0_r8 + rbuf2d = 0.0_r8 + do i=1,ncols + if ( iam == owners(i) ) then + do k = 1,nlevs + sbuf2d(k,i) = hitems(f)%hbuf( fdims(i), k, ldims(i) ) + enddo + endif + enddo + call mpi_allreduce(sbuf2d,rbuf2d,ncols*nlevs,mpi_real8, mpi_sum, mpicom, ierr) + buf2d(:,:) = real(rbuf2d(:,:),r4) + ierr = pio_put_var(File, vardesc, (/1,nfils/),(/nlevs,ncols/), buf2d(:,:)) + endif - cnt = 0 - buf = fillvalue - dof = 0 + endif - do i = 1,ncols - do k = 1,field%numlev - cnt = cnt+1 - if ( iam == owners(i) ) then - buf(cnt) = hitem%hbuf( fdims(i), k, ldims(i) ) - dof(cnt) = cnt - endif - enddo enddo - call pio_setframe(File, vardesc, int(-1,kind=PIO_OFFSET_KIND)) - - call pio_initdecomp(sat_iosystem, pio_double, dimlens, dof, iodesc ) + call t_stopf ('sat_hist::dump_columns') - call pio_setframe(File, vardesc, int(nfils,kind=PIO_OFFSET_KIND)) + end subroutine dump_columns - call pio_write_darray(File, vardesc, iodesc, buf, ierr, fillval=fillvalue) +!------------------------------------------------------------------------------- +! scan the fields for possible different decompositions +!------------------------------------------------------------------------------- + subroutine scan_flds( tape, nflds & + , has_phys_srf_flds, has_phys_lev_flds, has_phys_ilev_flds & + , has_dyn_srf_flds, has_dyn_lev_flds, has_dyn_ilev_flds ) + use cam_history_support, only : active_entry + use phys_grid, only: phys_decomp + use dyn_grid, only: dyn_decomp - call pio_freedecomp(sat_iosystem, iodesc) + type(active_entry), intent(in) :: tape + integer, intent(in) :: nflds + logical, save :: flds_scanned + logical, intent(out) :: has_phys_srf_flds + logical, intent(out) :: has_phys_lev_flds + logical, intent(out) :: has_phys_ilev_flds + logical, intent(out) :: has_dyn_srf_flds + logical, intent(out) :: has_dyn_lev_flds + logical, intent(out) :: has_dyn_ilev_flds + + integer :: f + character(len=cl) :: msg1, msg2 + + if (flds_scanned) return + + do f = 1,nflds + if ( tape%hlist(f)%field%decomp_type == phys_decomp ) then + if ( tape%hlist(f)%field%numlev == 1 ) then + has_phys_srf_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pver ) then + has_phys_lev_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pverp ) then + has_phys_ilev_flds = .true. + else + call endrun('sat_hist::scan_flds numlev error : '//tape%hlist(f)%field%name) + endif + elseif ( tape%hlist(f)%field%decomp_type == dyn_decomp ) then + if ( tape%hlist(f)%field%numlev == 1 ) then + has_dyn_srf_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pver ) then + has_dyn_lev_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pverp ) then + has_dyn_ilev_flds = .true. + else + call endrun('sat_hist::scan_flds numlev error : '//tape%hlist(f)%field%name) + endif + else + call endrun('sat_hist::scan_flds decomp_type error : '//tape%hlist(f)%field%name) + endif - deallocate( buf ) - deallocate( dof ) - deallocate( dimlens ) + ! Check that the only "mdim" is the vertical coordinate. + if (has_phys_srf_flds .or. has_phys_lev_flds .or. has_phys_ilev_flds .or. & + has_dyn_srf_flds .or. has_dyn_lev_flds .or. has_dyn_ilev_flds) then + ! The mdims pointer is unassociated on a restart. The restart initialization + ! should be fixed rather than requiring the check to make sure it is associated. + if (associated(tape%hlist(f)%field%mdims)) then + if (size(tape%hlist(f)%field%mdims) > 1) then + msg1 = 'sat_hist::scan_flds mdims error :'//tape%hlist(f)%field%name + msg2 = trim(msg1)//' has mdims in addition to the vertical coordinate.'//& + new_line('a')//' This is not currently supported.' + write(iulog,*) msg2 + call endrun(msg1) + end if + end if + end if - call t_stopf ('sat_hist::dump_columns') + enddo - end subroutine dump_columns + flds_scanned = .true. + end subroutine scan_flds !------------------------------------------------------------------------------- !------------------------------------------------------------------------------- subroutine read_next_position( ncols ) - use time_manager, only: get_curr_date, get_prev_date + use time_manager, only: get_curr_date use time_manager, only: set_time_float_from_date implicit none @@ -660,7 +739,7 @@ end subroutine read_next_position !------------------------------------------------------------------------------- subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils ) - use time_manager, only: get_nstep, get_curr_date, get_curr_time + use time_manager, only: get_curr_date, get_curr_time use cam_history_support, only : active_entry implicit none type(active_entry), intent(inout) :: tape @@ -671,9 +750,8 @@ subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils real(r8), intent(in) :: mod_dists(ncols * sathist_nclosest) integer, intent(in) :: nfils - integer :: t, ierr, i + integer :: ierr, i integer :: yr, mon, day ! year, month, and day components of a date - integer :: nstep ! current timestep number integer :: ncdate ! current date in integer format [yyyymmdd] integer :: ncsec ! current time of day [seconds] integer :: ndcur ! day component of current time @@ -686,7 +764,6 @@ subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils call t_startf ('sat_hist::write_record_coord') - nstep = get_nstep() call get_curr_date(yr, mon, day, ncsec) ncdate = yr*10000 + mon*100 + day call get_curr_time(ndcur, nscur) From a2fb3e7af1b9bb39db6531b87e735240ae973c50 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Tue, 24 Sep 2024 11:11:50 -0500 Subject: [PATCH 094/366] Correct ocean conservation check settings --- .../namelist_defaults_mpaso.xml | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index fd278561665..4f9b5d8fe4d 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -1232,37 +1232,10 @@ .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. 'dt' 'conservationCheckOutput' .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. .true. 'conservationCheckRestart' From beca87f8cece8ad0b650ee4ee3587cd9e2b2f2b9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 25 Sep 2024 04:29:47 -0500 Subject: [PATCH 095/366] Change conservationCheckOutput to append clobber mode --- components/mpas-ocean/cime_config/buildnml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 8f5193c334b..1b413efbb86 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -1156,7 +1156,7 @@ def buildnml(case, caseroot, compname): lines.append(' filename_interval="00-01-00_00:00:00"') lines.append(' reference_time="01-01-01_00:00:00"') lines.append(' output_interval="00-00-01_00:00:00"') - lines.append(' clobber_mode="truncate"') + lines.append(' clobber_mode="append"') lines.append(' packages="conservationCheckAMPKG">') lines.append('') lines.append('') From ff5aa6f2732488223737315fa464f7ccc933628f Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 25 Sep 2024 05:02:08 -0500 Subject: [PATCH 096/366] Add a stealth ERS test for conservation check AM --- cime_config/tests.py | 1 + .../testmods_dirs/mpaso/conservation_check/README | 8 ++++++++ .../testmods_dirs/mpaso/conservation_check/shell_commands | 2 ++ .../testmods_dirs/mpaso/conservation_check/user_nl_mpaso | 1 + 4 files changed, 12 insertions(+) create mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README create mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands create mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b8397..c24f24e22f7 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -266,6 +266,7 @@ "SMS_D_Ld1.T62_oQU240wLI.GMPAS-IAF-PISMF.mpaso-impl_top_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-harmonic_mean_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-upwind_advection", + "ERS_D.T62_oQU240.GMPAS-IAF.mpaso-conservation_check", ) }, diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README new file mode 100644 index 00000000000..9e13f46ced0 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README @@ -0,0 +1,8 @@ +This testdef is used to test the conservation check analysis member, which +wasintroduced in MPAS-Ocean PR #4521 and has been made a stealth feature in +#6643. This test turns on the consevation check analysis member by setting: + + config_AM_conservationCheck_enable = .true. + +Then, it enables the use of MPAS-Ocean history files in testing by running +a sed command to modify env_archive.xml. \ No newline at end of file diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands new file mode 100644 index 00000000000..67789cfe69c --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands @@ -0,0 +1,2 @@ +# include mpas-ocean outputs in testing +sed -i 's#compclass="ocn" exclude_testing="true"#compclass="ocn" exclude_testing="false"#g' env_archive.xml diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso new file mode 100644 index 00000000000..c57241a9fcf --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso @@ -0,0 +1 @@ + config_AM_conservationCheck_enable = .true. From 663e88019b52561bb4d00963c78e05b114554272 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Wed, 25 Sep 2024 11:38:44 -0500 Subject: [PATCH 097/366] Update stealth test based on review --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index c24f24e22f7..c707f607690 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -266,7 +266,7 @@ "SMS_D_Ld1.T62_oQU240wLI.GMPAS-IAF-PISMF.mpaso-impl_top_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-harmonic_mean_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-upwind_advection", - "ERS_D.T62_oQU240.GMPAS-IAF.mpaso-conservation_check", + "ERS_Ld5_D.T62_oQU240.GMPAS-IAF.mpaso-conservation_check", ) }, From a60ae936ccdefeab3be8801d46613549faf8ce3e Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 25 Sep 2024 11:54:05 -0500 Subject: [PATCH 098/366] add fixme note, handle error, and update test - Updated default filename spec - Handled error from put call - Added note to improve code - Updated test to ERS --- cime_config/tests.py | 2 +- .../eam/sathist_F2010/user_nl_eam | 2 +- components/eam/src/control/sat_hist.F90 | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index c3d7dc75740..d967ce311ef 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -207,7 +207,7 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", - "SMS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", + "ERS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam index 146e2da835c..bab3fc6f619 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam @@ -1,7 +1,7 @@ &satellite_options_nl sathist_mfilt = 10000, sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/sathist_master_19700410-20150419_c20150616.nc' - sathist_hfilename_spec = '%c.sathist_master_19700410-20150419_c20150616.%y-%m-%d-%s.nc' + sathist_hfilename_spec = '%c.eam.h9.sathist.%y-%m-%d-%s.nc' sathist_nclosest = 1 sathist_ntimestep = 1 sathist_fincl = 'T', 'PS' diff --git a/components/eam/src/control/sat_hist.F90 b/components/eam/src/control/sat_hist.F90 index af8e330e557..77b902babdf 100644 --- a/components/eam/src/control/sat_hist.F90 +++ b/components/eam/src/control/sat_hist.F90 @@ -125,7 +125,7 @@ subroutine sat_hist_readnl(nlfile, hfilename_spec, mfilt, fincl, nhtfrq, avgflag ! set defaults sathist_track_infile = ' ' - sathist_hfilename_spec = '%c.cam' // trim(inst_suffix) // '.hs.%y-%m-%d-%s.nc' + sathist_hfilename_spec = '%c.eam.hs.' // trim(inst_suffix) // '.%y-%m-%d-%s.nc' sathist_fincl(:) = ' ' sathist_mfilt = 100000 sathist_nclosest = 1 @@ -522,6 +522,13 @@ subroutine sat_hist_write( tape , nflds, nfils) end subroutine sat_hist_write !------------------------------------------------------------------------------- +! FIXME extra work > +! dump_columns routine is doing unnecessary extra work serially +! this happens because there is an unneeded mpi_allreduce call +! and then the gathered data is written in a serial manner; this +! could be improved by avoiding the mpi_allreduce call, and then +! writing local data out using pio_write_darray, which is parallel +! FIXME extra work < subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, owners, decomp ) use cam_history_support, only: field_info, hentry, fillvalue use pio, only: pio_setframe, pio_offset_kind @@ -565,9 +572,14 @@ subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, sbuf1d(i) = hitems(f)%hbuf( fdims(i), 1, ldims(i) ) endif enddo + ! FIXME extra work: unnecessary mpi call, then serial write + ! FIXME extra work: can use pio_write_darray on local data instead call mpi_allreduce(sbuf1d,rbuf1d,ncols,mpi_real8, mpi_sum, mpicom, ierr) buf1d(:) = real(rbuf1d(:),r4) ierr = pio_put_var(File, vardesc, (/nfils/),(/ncols/), buf1d(:)) + if ( ierr /= PIO_NOERR ) then + call endrun('sat_hist::dump_columns: pio_put_var error') + endif else sbuf2d = 0.0_r8 rbuf2d = 0.0_r8 @@ -578,9 +590,14 @@ subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, enddo endif enddo + ! FIXME extra work: unnecessary mpi call, then serial write + ! FIXME extra work: can use pio_write_darray on local data instead call mpi_allreduce(sbuf2d,rbuf2d,ncols*nlevs,mpi_real8, mpi_sum, mpicom, ierr) buf2d(:,:) = real(rbuf2d(:,:),r4) ierr = pio_put_var(File, vardesc, (/1,nfils/),(/nlevs,ncols/), buf2d(:,:)) + if ( ierr /= PIO_NOERR ) then + call endrun('sat_hist::dump_columns: pio_put_var error') + endif endif endif From 448f4f18f5b71c5cd694ce7764fcc3ea0a5bb9dd Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 25 Sep 2024 15:16:23 -0500 Subject: [PATCH 099/366] Fix sed command --- .../testmods_dirs/mpaso/conservation_check/shell_commands | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands index 67789cfe69c..62c82542595 100644 --- a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands @@ -1,2 +1,3 @@ # include mpas-ocean outputs in testing -sed -i 's#compclass="ocn" exclude_testing="true"#compclass="ocn" exclude_testing="false"#g' env_archive.xml +sed 's|compname="mpaso" exclude_testing="true"|compname="mpaso" exclude_testing="false"|' env_archive.xml > tmp_env_archive.xml +mv tmp_env_archive.xml env_archive.xml From 6944e95cd34004d942d1345b32c44917e6738904 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Wed, 25 Sep 2024 15:49:47 -0500 Subject: [PATCH 100/366] Homme(xx): Flush the log before calling MPI_Abort in Homme's abortmp. This is an attempt to assure atm.log (EAM) and homme_atm.log (EAMxx) get full output when the dycore calls MPI_Abort. --- components/homme/src/share/parallel_mod.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/components/homme/src/share/parallel_mod.F90 b/components/homme/src/share/parallel_mod.F90 index 69016a8a063..49ff71779a8 100644 --- a/components/homme/src/share/parallel_mod.F90 +++ b/components/homme/src/share/parallel_mod.F90 @@ -274,6 +274,7 @@ subroutine abortmp(string) #endif #endif character*(*) string + call flush(iulog) #ifdef CAM call endrun(string) #else From fac8382406db337c15166585a75b28f79bf53adb Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 23 Sep 2024 12:01:53 -0400 Subject: [PATCH 101/366] add gh/ci eamxx standalone testing --- .github/workflows/eamxx-gh-ci-standalone.yml | 69 +++++++++++++++++++ components/eamxx/cmake/BuildCprnc.cmake | 62 ++++++++++------- .../eamxx/cmake/machine-files/ghci-oci.cmake | 13 ++++ components/eamxx/scripts/machines_specs.py | 4 ++ 4 files changed, 123 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/eamxx-gh-ci-standalone.yml diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml new file mode 100644 index 00000000000..fbbdf813581 --- /dev/null +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -0,0 +1,69 @@ +name: gh-standalone + +on: + pull_request: + branches: [ master ] + paths: + # first, yes to these + - '.github/workflows/eamxx-gh-ci-standalone.yml' + - 'cime_config/**' + - 'components/eam/**' + - 'components/eamxx/**' + - 'components/elm/**' + - 'driver-moab/**' + - 'driver-mct/**' + - 'components/homme/**' + # second, no to these + - '!components/eam/docs/**' + - '!components/eam/mkdocs.yml' + - '!components/eamxx/docs/**' + - '!components/eamxx/mkdocs.yml' + - '!components/elm/docs/**' + - '!components/elm/mkdocs.yml' + + workflow_dispatch: + +jobs: + + ci: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test: + # TODO: add opt, dbg, etc. here once we stabilize testing + # TODO: note that currently, there is a fail in atm_proc test + # TODO: components/eamxx/src/share/tests/atm_process_tests.cpp + # TODO: REQUIRE (dag.has_unmet_dependencies()); + # TODO: but only on some machines... + - sp + container: + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.2 + + steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + show-progress: false + submodules: recursive + - + name: standalone + env: + SHELL: sh + run: | + # TODO: get rid of this extra line if we can? + git config --global safe.directory '*' + ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} + # TODO: add logging... + # - + # name: Artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ always() }} + # with: + # name: ${{ matrix.test }} + # path: | + # /projects/e3sm/scratch/${{ matrix.test }}*/TestStatus.log + # /projects/e3sm/scratch/${{ matrix.test }}*/bld/*.bldlog.* + # /projects/e3sm/scratch/${{ matrix.test }}*/run/*.log.* + # /projects/e3sm/scratch/${{ matrix.test }}*/run/*.cprnc.out diff --git a/components/eamxx/cmake/BuildCprnc.cmake b/components/eamxx/cmake/BuildCprnc.cmake index 2f4f1f00a36..287956c5a9d 100644 --- a/components/eamxx/cmake/BuildCprnc.cmake +++ b/components/eamxx/cmake/BuildCprnc.cmake @@ -8,32 +8,44 @@ include (EkatUtils) macro(BuildCprnc) - # Make sure this is built only once - if (NOT TARGET cprnc) - if (SCREAM_CIME_BUILD) - string (CONCAT MSG - "WARNING! By default, scream should not build tests in a CIME build,\n" - "and cprnc should only be built by scream in case tests are enabled.\n" - "If you explicitly requested tests to be on in a CIME build,\n" - "then you can discard this warning. Otherwise, please, contact developers.\n") - message("${MSG}") - endif() - set(BLDROOT ${PROJECT_BINARY_DIR}/externals/cprnc) - file(WRITE ${BLDROOT}/Macros.cmake - " - set(SCC ${CMAKE_C_COMPILER}) - set(SFC ${CMAKE_Fortran_COMPILER}) - set(FFLAGS \"${CMAKE_Fortran_FLAGS}\") - set(NETCDF_PATH ${NetCDF_Fortran_PATH}) - " - ) - set(SRC_ROOT ${SCREAM_BASE_DIR}/../..) - add_subdirectory(${SRC_ROOT}/cime/CIME/non_py/cprnc ${BLDROOT}) - EkatDisableAllWarning(cprnc) - - set(CPRNC_BINARY ${BLDROOT}/cprnc CACHE INTERNAL "") - + # TODO: handle this more carefully and more gracefully in the future + # TODO: For now, it is just a hack to get going... + # find cprnc defined in machine entries + set(CCSM_CPRNC $ENV{CCSM_CPRNC}) + if(EXISTS "${CCSM_CPRNC}") + message(STATUS "Path ${CCSM_CPRNC} exists, so we will use it") + set(CPRNC_BINARY ${CCSM_CPRNC} CACHE INTERNAL "") configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) + else() + message(WARNING "Path ${CCSM_CPRNC} does not exist, so we will try to build it") + # Make sure this is built only once + if (NOT TARGET cprnc) + if (SCREAM_CIME_BUILD) + string (CONCAT MSG + "WARNING! By default, scream should not build tests in a CIME build,\n" + "and cprnc should only be built by scream in case tests are enabled.\n" + "If you explicitly requested tests to be on in a CIME build,\n" + "then you can discard this warning. Otherwise, please, contact developers.\n") + message("${MSG}") + endif() + set(BLDROOT ${PROJECT_BINARY_DIR}/externals/cprnc) + file(WRITE ${BLDROOT}/Macros.cmake + " + set(SCC ${CMAKE_C_COMPILER}) + set(SFC ${CMAKE_Fortran_COMPILER}) + set(FFLAGS \"${CMAKE_Fortran_FLAGS}\") + set(NETCDF_PATH ${NetCDF_Fortran_PATH}) + " + ) + set(SRC_ROOT ${SCREAM_BASE_DIR}/../..) + add_subdirectory(${SRC_ROOT}/cime/CIME/non_py/cprnc ${BLDROOT}) + EkatDisableAllWarning(cprnc) + + set(CPRNC_BINARY ${BLDROOT}/cprnc CACHE INTERNAL "") + + configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in + ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) + endif() endif() endmacro() diff --git a/components/eamxx/cmake/machine-files/ghci-oci.cmake b/components/eamxx/cmake/machine-files/ghci-oci.cmake index 86a2fb1d530..85eabbaa848 100644 --- a/components/eamxx/cmake/machine-files/ghci-oci.cmake +++ b/components/eamxx/cmake/machine-files/ghci-oci.cmake @@ -1,2 +1,15 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() + +set(CMAKE_Fortran_FLAGS "-Wno-maybe-uninitialized -Wno-unused-dummy-argument -fallow-argument-mismatch" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-fvisibility-inlines-hidden -fmessage-length=0 -Wno-use-after-free -Wno-unused-variable -Wno-maybe-uninitialized" CACHE STRING "" FORCE) + +# TODO: figure out a better way to handle this, e.g., +# TODO: --map-by ppr:1:node:pe=1 doesn't work with mpich, +# TODO: but -map-by core:1:numa:hwthread=1 may work well? +# TODO: this will need to be handled in EKAT at some point +set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "-np") + +# TODO: hack in place to get eamxx to recognize CPRNC +# TODO: See note in BuildCprnc.cmake... +set(ENV{CCSM_CPRNC} "/usr/local/packages/bin/cprnc") diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 9536d415c0d..0b0567c3cce 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,6 +82,10 @@ ["mpicxx","mpifort","mpicc"], "", ""), + "ghci-oci" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-oci_gnu)"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), From 89b7ade03312aac5aa8ece869cebc356919741c6 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 24 Sep 2024 11:42:17 -0400 Subject: [PATCH 102/366] fix pl semanitics --- components/eamxx/src/share/tests/atm_process_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/tests/atm_process_tests.cpp b/components/eamxx/src/share/tests/atm_process_tests.cpp index 01985c33c6d..fce25cd28e2 100644 --- a/components/eamxx/src/share/tests/atm_process_tests.cpp +++ b/components/eamxx/src/share/tests/atm_process_tests.cpp @@ -443,7 +443,7 @@ TEST_CASE("atm_proc_dag", "") { using strvec_t = std::vector; auto params = create_test_params(); - auto p1 = params.sublist("BarBaz"); + auto& p1 = params.sublist("BarBaz"); // Make sure there's a missing piece (whatever Baz computes); p1.set("atm_procs_list",{"Bar"}); From cdaa5c4b8c4af6eeae2f11a487a185e96d7dbae6 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 24 Sep 2024 18:09:01 -0400 Subject: [PATCH 103/366] only test for eamxx, save logs --- .github/workflows/eamxx-gh-ci-standalone.yml | 36 ++++++-------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index fbbdf813581..4a194a89106 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -6,20 +6,12 @@ on: paths: # first, yes to these - '.github/workflows/eamxx-gh-ci-standalone.yml' - - 'cime_config/**' - - 'components/eam/**' + - 'cime_config/machine/config_machines.xml' - 'components/eamxx/**' - - 'components/elm/**' - - 'driver-moab/**' - - 'driver-mct/**' - 'components/homme/**' # second, no to these - - '!components/eam/docs/**' - - '!components/eam/mkdocs.yml' - '!components/eamxx/docs/**' - '!components/eamxx/mkdocs.yml' - - '!components/elm/docs/**' - - '!components/elm/mkdocs.yml' workflow_dispatch: @@ -31,12 +23,8 @@ jobs: fail-fast: false matrix: test: - # TODO: add opt, dbg, etc. here once we stabilize testing - # TODO: note that currently, there is a fail in atm_proc test - # TODO: components/eamxx/src/share/tests/atm_process_tests.cpp - # TODO: REQUIRE (dag.has_unmet_dependencies()); - # TODO: but only on some machines... - sp + - opt container: image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.2 @@ -55,15 +43,11 @@ jobs: # TODO: get rid of this extra line if we can? git config --global safe.directory '*' ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} - # TODO: add logging... - # - - # name: Artifacts - # uses: actions/upload-artifact@v4 - # if: ${{ always() }} - # with: - # name: ${{ matrix.test }} - # path: | - # /projects/e3sm/scratch/${{ matrix.test }}*/TestStatus.log - # /projects/e3sm/scratch/${{ matrix.test }}*/bld/*.bldlog.* - # /projects/e3sm/scratch/${{ matrix.test }}*/run/*.log.* - # /projects/e3sm/scratch/${{ matrix.test }}*/run/*.cprnc.out + - + name: Artifacts + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: ${{ matrix.test }} + path: | + components/eamxx/ctest-build/*/Testing/Temporary/Last*.log From 2982d4af4f10b5a09628927bbfe54b58e8628062 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 26 Sep 2024 01:04:28 -0500 Subject: [PATCH 104/366] Update conservation_check test to remove sed command --- .../testdefs/testmods_dirs/mpaso/conservation_check/README | 6 ++++-- .../testmods_dirs/mpaso/conservation_check/shell_commands | 3 --- 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README index 9e13f46ced0..ef7ef254823 100644 --- a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README @@ -4,5 +4,7 @@ wasintroduced in MPAS-Ocean PR #4521 and has been made a stealth feature in config_AM_conservationCheck_enable = .true. -Then, it enables the use of MPAS-Ocean history files in testing by running -a sed command to modify env_archive.xml. \ No newline at end of file +However, it shoudl be noted that MPAS-Ocean history files are not currently +included in E3SM testing so non-BFB results will not be detected unless one +manually changes to 'compname="mpaso" exclude_testing="false"' in the file +cime_config/config_archive.xml. diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands deleted file mode 100644 index 62c82542595..00000000000 --- a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands +++ /dev/null @@ -1,3 +0,0 @@ -# include mpas-ocean outputs in testing -sed 's|compname="mpaso" exclude_testing="true"|compname="mpaso" exclude_testing="false"|' env_archive.xml > tmp_env_archive.xml -mv tmp_env_archive.xml env_archive.xml From 87182bba27827f59b0bda3df94cdb6553d18a928 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 26 Sep 2024 20:03:55 +0200 Subject: [PATCH 105/366] Fix readme for the new test --- .../testdefs/testmods_dirs/mpaso/conservation_check/README | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README index ef7ef254823..fb85b185fd4 100644 --- a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README @@ -4,7 +4,5 @@ wasintroduced in MPAS-Ocean PR #4521 and has been made a stealth feature in config_AM_conservationCheck_enable = .true. -However, it shoudl be noted that MPAS-Ocean history files are not currently -included in E3SM testing so non-BFB results will not be detected unless one -manually changes to 'compname="mpaso" exclude_testing="false"' in the file -cime_config/config_archive.xml. +However, it should be noted that MPAS-Ocean history files are not currently +included in E3SM testing so non-BFB results will not be detected. From 712d71fd7cc7eef3684b970f9321624d3df2b73c Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Thu, 26 Sep 2024 14:44:08 -0500 Subject: [PATCH 106/366] Zero aerosol diagnostics to prevent NaNs during bgc only runs BFB --- .../src/shared/mpas_seaice_icepack.F | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F index fa3b28c92c8..9dc6f06ec34 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F @@ -15002,7 +15002,8 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) type(MPAS_pool_type), pointer :: & biogeochemistryPool, & - diagnostics_biogeochemistryPool + diagnostics_biogeochemistryPool, & + tracers_aggregatePool ! biogeochemistry real(kind=RKIND), dimension(:), pointer :: & @@ -15050,7 +15051,12 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBiologySnow, & bgridPorosityIceCell, & bgridSalinityIceCell, & - bgridTemperatureIceCell + bgridTemperatureIceCell, & + verticalBCTotalIceCell, & + verticalDustTotalIceCell, & + verticalBCTotalSnowCell, & + verticalDustTotalSnowCell + real(kind=RKIND), dimension(:,:,:), pointer :: & bioTracerShortwave @@ -15080,6 +15086,7 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistryPool) call MPAS_pool_get_subpool(block % structs, "diagnostics_biogeochemistry", diagnostics_biogeochemistryPool) + call MPAS_pool_get_subpool(block % structs, "tracers_aggregate", tracers_aggregatePool) if (config_use_vertical_tracers) then call MPAS_pool_get_array(biogeochemistryPool, "primaryProduction", primaryProduction) @@ -15132,6 +15139,10 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCSnow", totalVerticalBCSnow) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalBCTotalIceCell", verticalBCTotalIceCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalDustTotalIceCell", verticalDustTotalIceCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalDustTotalSnowCell", verticalDustTotalSnowCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalBCTotalSnowCell", verticalBCTotalSnowCell) netBrineHeight = 0.0_RKIND @@ -15165,10 +15176,14 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBC1Snow = 0.0_RKIND totalVerticalBC2Snow = 0.0_RKIND totalVerticalBCSnow = 0.0_RKIND - totalVerticalBCSnow = 0.0_RKIND + totalVerticalBCIce = 0.0_RKIND totalVerticalDustSnow = 0.0_RKIND totalVerticalAlgaeCarbonIce = 0.0_RKIND totalVerticalDOCLabileIce = 0.0_RKIND + verticalBCTotalIceCell = 0.0_RKIND + verticalDustTotalIceCell = 0.0_RKIND + verticalBCTotalSnowCell = 0.0_RKIND + verticalDustTotalSnowCell = 0.0_RKIND endif From c30f09b60957a6e41fa928d9a83a52929a500f75 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Thu, 26 Sep 2024 16:30:46 -0500 Subject: [PATCH 107/366] rename ww3 obstructions/shadow files to include ".rtd." --- components/ww3/bld/build-namelist | 4 ++-- components/ww3/cime_config/buildnml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/ww3/bld/build-namelist b/components/ww3/bld/build-namelist index 314949671c4..915e6b0bf2a 100755 --- a/components/ww3/bld/build-namelist +++ b/components/ww3/bld/build-namelist @@ -398,8 +398,8 @@ if ($NML_TYPE eq "ww3_grid_nml") { add_default($nl, 'icedisp'); add_default($nl, 'rwndc'); - add_default($nl, 'uostfilelocal', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_local.${WAV_GRID}${WAV_SPEC}.in'"); - add_default($nl, 'uostfileshadow', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_shadow.${WAV_GRID}${WAV_SPEC}.in'"); + add_default($nl, 'uostfilelocal', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_local.${WAV_GRID}${WAV_SPEC}.rtd.in'"); + add_default($nl, 'uostfileshadow', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_shadow.${WAV_GRID}${WAV_SPEC}.rtd.in'"); } diff --git a/components/ww3/cime_config/buildnml b/components/ww3/cime_config/buildnml index 8e7cd2ea84e..f550d032188 100755 --- a/components/ww3/cime_config/buildnml +++ b/components/ww3/cime_config/buildnml @@ -66,8 +66,8 @@ def buildnml(case, caseroot, compname): input_list.write("mesh = {}/wav/ww3/{}.msh\n".format(din_loc_root,wav_grid)) input_list.write("stations = {}/wav/ww3/stations.txt\n".format(din_loc_root)) - input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.in\n".format(din_loc_root,wav_grid,wav_spec)) - input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.in\n".format(din_loc_root,wav_grid,wav_spec)) + input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) + input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) #-------------------------------------------------------------------- # Invoke ww3 build-namelist - output will go in $CASEBUILD/ww3conf From dd108ac849e64ad2efdc72419faa1aca23c595a3 Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Thu, 26 Sep 2024 17:39:50 -0400 Subject: [PATCH 108/366] Fix initialization of doubles --- components/elm/src/biogeochem/CH4Mod.F90 | 2 +- .../elm/src/biogeochem/VerticalProfileMod.F90 | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/components/elm/src/biogeochem/CH4Mod.F90 b/components/elm/src/biogeochem/CH4Mod.F90 index 3b3c1fa8061..a5cbd2f041c 100644 --- a/components/elm/src/biogeochem/CH4Mod.F90 +++ b/components/elm/src/biogeochem/CH4Mod.F90 @@ -223,7 +223,7 @@ subroutine InitAllocate(this, bounds) ! Allocate module variables and data structures ! ! !USES: - use shr_infnan_mod, only: spval => shr_infnan_nan, assignment(=) + use shr_infnan_mod, only: nan => shr_infnan_nan, assignment(=) use elm_varpar , only: nlevgrnd ! ! !ARGUMENTS: diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index 3f5ced72f33..d5bd1d1c83f 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -25,9 +25,9 @@ module VerticalProfileMod logical , public :: exponential_rooting_profile = .true. logical , public :: pftspecific_rootingprofile = .true. ! how steep profile is for root C inputs (1/ e-folding depth) (1/m) - real(r8), public :: rootprof_exp = 3. + real(r8), public :: rootprof_exp = 3._r8 ! how steep profile is for surface components (1/ e_folding depth) (1/m) - real(r8), public :: surfprof_exp = 10. + real(r8), public :: surfprof_exp = 10._r8 !----------------------------------------------------------------------- contains @@ -83,7 +83,7 @@ subroutine decomp_vertprofiles(bounds, & real(r8) :: ndep_prof_sum real(r8) :: nfixation_prof_sum real(r8) :: pdep_prof_sum - real(r8) :: delta = 1.e-10 + real(r8) :: delta = 1.e-10_r8 character(len=32) :: subname = 'decomp_vertprofiles' !----------------------------------------------------------------------- @@ -294,9 +294,9 @@ subroutine decomp_vertprofiles(bounds, & ! check to make sure integral of all profiles = 1. do fc = 1,num_soilc c = filter_soilc(fc) - ndep_prof_sum = 0. - nfixation_prof_sum = 0. - pdep_prof_sum = 0. + ndep_prof_sum = 0._r8 + nfixation_prof_sum = 0._r8 + pdep_prof_sum = 0._r8 do j = 1, nlevdecomp ndep_prof_sum = ndep_prof_sum + ndep_prof(c,j) * dzsoi_decomp(j) nfixation_prof_sum = nfixation_prof_sum + nfixation_prof(c,j) * dzsoi_decomp(j) @@ -324,10 +324,10 @@ subroutine decomp_vertprofiles(bounds, & do fp = 1,num_soilp p = filter_soilp(fp) - froot_prof_sum = 0. - croot_prof_sum = 0. - leaf_prof_sum = 0. - stem_prof_sum = 0. + froot_prof_sum = 0._r8 + croot_prof_sum = 0._r8 + leaf_prof_sum = 0._r8 + stem_prof_sum = 0._r8 do j = 1, nlevdecomp froot_prof_sum = froot_prof_sum + froot_prof(p,j) * dzsoi_decomp(j) croot_prof_sum = croot_prof_sum + croot_prof(p,j) * dzsoi_decomp(j) From 387c32722f6dbe72f8414a34b8388df671d6673a Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Thu, 26 Sep 2024 17:02:44 -0500 Subject: [PATCH 109/366] add tests to wav_developer test suite --- cime_config/tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b8397..98d11caea88 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -810,7 +810,10 @@ "e3sm_wav_developer" : { "time" : "0:45:00", "tests" : ( - "ERS.T62_oEC60to30v3_wQU225EC60to30.GMPAS-IAF-WW3", + "ERS.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "PEM.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "PET.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "SMS_D_Ln3.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", ) }, From e9c8bcf87933614d54cb1b83b55f9d4cf4db1ef3 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Thu, 26 Sep 2024 18:33:36 -0400 Subject: [PATCH 110/366] remove test from the e3sm integration test suite --- cime_config/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index d967ce311ef..1cbf28b8397 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -207,7 +207,6 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", - "ERS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, From f4d95ed6ad7d81ebbb19dbeaf6be259de2ceac17 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 26 Sep 2024 18:18:23 -0500 Subject: [PATCH 111/366] Add missing omp private variables for data subglacial runoff --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index a2d6586a7d6..1777e1f6d1e 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3540,6 +3540,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & !$omp parallel !$omp do schedule(runtime) & !$omp private(kmin, fracAbsorbed, fracAbsorbedRunoff, fracAbsorbedSubglacialRunoff, & + !$omp zTop, k, zBot, transmissionCoeffTop, transmissionCoeffBot, & !$omp sumSurfaceStressSquared, i, iEdge) do iCell = 1, nCells From 8c905eb15766b28ccbaf9359a0970bcba627c528 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Fri, 27 Sep 2024 11:57:47 -0500 Subject: [PATCH 112/366] Make bgc balance_check_tolerance a nml variable and increase the default --- components/elm/bld/namelist_files/namelist_definition.xml | 6 ++++++ .../elm/src/biogeochem/EcosystemBalanceCheckMod.F90 | 7 +++++-- components/elm/src/main/controlMod.F90 | 8 +++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 7063d7a1105..1ba59903170 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -139,6 +139,12 @@ If TRUE consider priority of plant to get a fraction of symbiotic N fixation and P phosphatase + +BGC balance check tolerance +Default 1.0e-7 hardwired + + Set date for beginning of adding temperature to atmospheric forcing diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index ffe1be5d909..dab999478eb 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -48,7 +48,8 @@ module EcosystemBalanceCheckMod implicit none save private - real(r8), parameter :: balance_check_tolerance = 1e-8_r8 + + real(r8), public :: balance_check_tolerance = 1e-7_r8 ! ! !PUBLIC MEMBER FUNCTIONS: public :: BeginColCBalance @@ -64,6 +65,8 @@ module EcosystemBalanceCheckMod public :: EndGridCBalanceAfterDynSubgridDriver public :: EndGridNBalanceAfterDynSubgridDriver public :: EndGridPBalanceAfterDynSubgridDriver + + !----------------------------------------------------------------------- contains @@ -498,7 +501,7 @@ subroutine ColNBalanceCheck(bounds, & ! here is '-' adjustment. It says that the adding to PF decomp n pools was less. end if - if (abs(col_errnb(c)) > 1e-8_r8) then + if (abs(col_errnb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index 1d8f48d7cc7..7775a59dbbf 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -55,6 +55,7 @@ module controlMod use elm_varctl , only: const_climate_hist use elm_varctl , only: use_top_solar_rad use elm_varctl , only: snow_shape, snicar_atm_type, use_dust_snow_internal_mixing + use EcosystemBalanceCheckMod, only: balance_check_tolerance ! ! !PUBLIC TYPES: @@ -181,6 +182,10 @@ subroutine control_init( ) namelist /elm_inparm/ & NFIX_PTASE_plant + ! BGC balance check + namelist /elm_inparm/ & + balance_check_tolerance + ! For experimental manipulations namelist /elm_inparm/ & startdate_add_temperature @@ -325,7 +330,7 @@ subroutine control_init( ) namelist /elm_mosart/ & lnd_rof_coupling_nstep - + namelist /elm_inparm/ & snow_shape, snicar_atm_type, use_dust_snow_internal_mixing @@ -796,6 +801,7 @@ subroutine control_spmd() call mpi_bcast (forest_fert_exp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (ECA_Pconst_RGspin, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (NFIX_PTASE_plant, 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast (balance_check_tolerance, 1, MPI_REAL8, 0, mpicom, ier) call mpi_bcast (use_pheno_flux_limiter, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (startdate_add_temperature, 1, MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (startdate_add_co2, 1, MPI_CHARACTER, 0, mpicom, ier) From c1631a81ecad43dfaf4dec371aec14eb0b145ea7 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Fri, 27 Sep 2024 12:12:57 -0500 Subject: [PATCH 113/366] Remove extra blank lines --- components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index dab999478eb..879bd6cc64d 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -65,8 +65,6 @@ module EcosystemBalanceCheckMod public :: EndGridCBalanceAfterDynSubgridDriver public :: EndGridNBalanceAfterDynSubgridDriver public :: EndGridPBalanceAfterDynSubgridDriver - - !----------------------------------------------------------------------- contains From 9a66d99c9438922e1a6482f5eda7c9bab5038135 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 26 Sep 2024 19:24:23 -0700 Subject: [PATCH 114/366] reinstate integration test with smaller sat file --- cime_config/tests.py | 1 + .../testmods_dirs/eam/sathist_F2010/shell_commands | 2 +- .../testmods_dirs/eam/sathist_F2010/user_nl_eam | 2 +- components/eam/src/control/sat_hist.F90 | 10 ++++++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b8397..d967ce311ef 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -207,6 +207,7 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", + "ERS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands index 0dc0f9cc72c..92cb057059a 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands @@ -1,3 +1,3 @@ #!/bin/bash -./xmlchange RUN_STARTDATE=2015-01-01 +./xmlchange RUN_STARTDATE=2018-01-01 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam index bab3fc6f619..e9b723f13b8 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam @@ -1,6 +1,6 @@ &satellite_options_nl sathist_mfilt = 10000, - sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/sathist_master_19700410-20150419_c20150616.nc' + sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/satellite_profilelist_orcas_to_socrates_c190208.nc' sathist_hfilename_spec = '%c.eam.h9.sathist.%y-%m-%d-%s.nc' sathist_nclosest = 1 sathist_ntimestep = 1 diff --git a/components/eam/src/control/sat_hist.F90 b/components/eam/src/control/sat_hist.F90 index 77b902babdf..17d2a92bc7b 100644 --- a/components/eam/src/control/sat_hist.F90 +++ b/components/eam/src/control/sat_hist.F90 @@ -722,8 +722,14 @@ subroutine read_next_position( ncols ) call read_buffered_datetime( datetime, i ) - if ( datetime>begdatetime .and. beg_ndx<0 ) beg_ndx = i - if ( datetime>enddatetime ) exit bnds_loop + if (datetime > begdatetime .and. beg_ndx < 0) then + beg_ndx = i + end if + + if (datetime > enddatetime) then + exit bnds_loop + end if + end_ndx = i enddo bnds_loop From f46455b88ad7f60e70d4322915353e080f31a717 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Fri, 27 Sep 2024 15:23:24 -0500 Subject: [PATCH 115/366] Rename the nml variable as bgc_balance_check_tolerance --- components/elm/bld/namelist_files/namelist_definition.xml | 2 +- components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 | 2 ++ components/elm/src/main/controlMod.F90 | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 1ba59903170..95927912014 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -139,7 +139,7 @@ If TRUE consider priority of plant to get a fraction of symbiotic N fixation and P phosphatase - BGC balance check tolerance Default 1.0e-7 hardwired diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index 879bd6cc64d..a2ad41bf665 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -49,7 +49,9 @@ module EcosystemBalanceCheckMod save private + ! This corersponds to namelist variable bgc_balance_check_tolerance real(r8), public :: balance_check_tolerance = 1e-7_r8 + ! ! !PUBLIC MEMBER FUNCTIONS: public :: BeginColCBalance diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index 7775a59dbbf..dc7fa975dbb 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -55,7 +55,7 @@ module controlMod use elm_varctl , only: const_climate_hist use elm_varctl , only: use_top_solar_rad use elm_varctl , only: snow_shape, snicar_atm_type, use_dust_snow_internal_mixing - use EcosystemBalanceCheckMod, only: balance_check_tolerance + use EcosystemBalanceCheckMod, only: bgc_balance_check_tolerance => balance_check_tolerance ! ! !PUBLIC TYPES: @@ -184,7 +184,7 @@ subroutine control_init( ) ! BGC balance check namelist /elm_inparm/ & - balance_check_tolerance + bgc_balance_check_tolerance ! For experimental manipulations namelist /elm_inparm/ & @@ -801,7 +801,7 @@ subroutine control_spmd() call mpi_bcast (forest_fert_exp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (ECA_Pconst_RGspin, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (NFIX_PTASE_plant, 1, MPI_LOGICAL, 0, mpicom, ier) - call mpi_bcast (balance_check_tolerance, 1, MPI_REAL8, 0, mpicom, ier) + call mpi_bcast (bgc_balance_check_tolerance, 1, MPI_REAL8, 0, mpicom, ier) call mpi_bcast (use_pheno_flux_limiter, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (startdate_add_temperature, 1, MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (startdate_add_co2, 1, MPI_CHARACTER, 0, mpicom, ier) From 39da2624d7320d55f095af72c89682aa583e7783 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 22 Aug 2023 10:09:08 +0200 Subject: [PATCH 116/366] Add SOwISC12to30E3r3 to MPAS-Ocean, MPAS-Seaice and ELM --- .../bld/namelist_files/namelist_definition.xml | 2 +- .../namelist_files/namelist_defaults_mpaso.xml | 18 ++++++++++++++++++ components/mpas-ocean/cime_config/buildnml | 14 ++++++++++++++ .../namelist_defaults_mpassi.xml | 4 ++++ components/mpas-seaice/cime_config/buildnml | 10 ++++++++++ 5 files changed, 47 insertions(+), 1 deletion(-) diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 7063d7a1105..83e7fcf9490 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -1491,7 +1491,7 @@ Representative concentration pathway for future scenarios [radiative forcing at + valid_values="USGS,gx3v7,gx1v6,navy,test,tx0.1v2,tx1v1,T62,TL319,cruncep,oEC60to30v3,oEC60to30v3wLI,ECwISC30to60E1r2,EC30to60E2r2,WC14to60E2r3,WCAtl12to45E2r4,SOwISC12to60E2r4,ECwISC30to60E2r1,oRRS18to6,oRRS18to6v3,oRRS15to5,oARRM60to10,oARRM60to6,ARRM10to60E2r1,oQU480,oQU240,oQU240wLI,oQU120,oRRS30to10v3,oRRS30to10v3wLI,360x720cru,NLDASww3a,NLDAS,tx0.1v2,ICOS10,IcoswISC30E3r5,IcosXISC30E3r7,RRSwISC6to18E3r5,SOwISC12to30E3r3"> Land mask description diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 4f9b5d8fe4d..3b93400f4e8 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -55,6 +55,7 @@ '00:02:00' '00:01:00' '00:05:00' +'00:10:00' 'split_explicit_ab2' 2 @@ -85,6 +86,7 @@ .true. .true. .true. +.true. -1.0 .false. 30.0e3 @@ -107,6 +109,7 @@ .true. .true. .true. +.true. 10.0 1000.0 1000.0 @@ -123,6 +126,7 @@ 77.0 38.5 100.0 +462.0 .false. 10.0 @@ -154,6 +158,7 @@ 5.46e07 6.83e06 3.2e09 +1.18e10 1.0 .false. 0.0 @@ -191,6 +196,7 @@ 'RossbyRadius' 'RossbyRadius' 'RossbyRadius' +'RossbyRadius' 20e3 30e3 30e3 @@ -222,6 +228,7 @@ 'N2_dependent' 'N2_dependent' 'N2_dependent' +'N2_dependent' 900.0 600.0 600.0 @@ -236,6 +243,7 @@ 600.0 600.0 600.0 +600.0 0.3 'constant' 300.0 @@ -249,6 +257,7 @@ 1.0 1.0 1.0 +1.0 0.13 1000.0 200.0 @@ -264,6 +273,7 @@ 'RossbyRadius' 'RossbyRadius' 'RossbyRadius' +'RossbyRadius' 20e3 30e3 30e3 @@ -416,6 +426,7 @@ 'pressure_only' 'pressure_only' 'pressure_only' +'pressure_only' 'Jenkins' .false. 10.0 @@ -435,6 +446,7 @@ 4.48e-3 4.48e-3 4.48e-3 +4.48e-3 1e-4 0.011 0.00295 @@ -448,6 +460,7 @@ 0.00295 0.00295 0.00295 +0.00295 3.1e-4 8.42e-5 8.42e-5 @@ -460,6 +473,7 @@ 8.42e-5 8.42e-5 8.42e-5 +8.42e-5 1.0 0.0 5e-2 @@ -492,6 +506,7 @@ 4.48e-3 4.48e-3 4.48e-3 +4.48e-3 1.0e-3 10.0 2.5e-3 @@ -579,6 +594,7 @@ '0000_00:00:02.5' '0000_00:00:01.25' '0000_00:00:05' +'0000_00:00:15' 2 .true. 2 @@ -626,6 +642,7 @@ .false. .false. .false. +.false. .false. .false. .false. @@ -1153,6 +1170,7 @@ .true. .true. .true. +.true. '0000-00-00_01:00:00' 'mocStreamfunctionOutput' .true. diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 1b413efbb86..819572f617c 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -406,6 +406,20 @@ def buildnml(case, caseroot, compname): if ocn_ismf == 'data': data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' + elif ocn_grid == 'SOwISC12to30E3r3': + decomp_date = '20240705' + decomp_prefix = 'partitions/mpas-o.graph.info.' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240705.nc' + analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' + ic_date = '20240705' + ic_prefix = 'mpaso.SOwISC12to30E3r3' + if ocn_ic_mode == 'spunup': + logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") + logger.warning(" But no file available for this grid.") + if ocn_ismf == 'data': + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240705.n' + + #-------------------------------------------------------------------- # Set OCN_FORCING = datm_forced_restoring if restoring file is available #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml index d62797de451..69294640ff6 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml @@ -31,6 +31,7 @@ 120.0 60.0 900.0 +1800.0 'noleap' '2000-01-01_00:00:00' 'none' @@ -87,6 +88,7 @@ 85.0 85.0 85.0 +85.0 75.0 85.0 85.0 @@ -102,6 +104,7 @@ -85.0 -85.0 -85.0 +-85.0 -85.0 -85.0 -85.0 @@ -170,6 +173,7 @@ 1 1 2 +1 true true 120 diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index c38ca08a31d..821b2d09d54 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -330,6 +330,16 @@ def buildnml(case, caseroot, compname): logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") + elif ice_grid == 'SOwISC12to30E3r3': + grid_date = '20240705' + grid_prefix = 'mpassi.SOwISC12to30E3r3' + decomp_date = '20240705' + decomp_prefix = 'partitions/mpas-seaice.graph.info.' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240705.nc' + if ice_ic_mode == 'spunup': + logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") + logger.warning(" But no file available for this grid.") + elif ice_grid == 'ICOS10': grid_date = '211015' grid_prefix = 'seaice.ICOS10' From 4ea7036711c4678361df326232b9e4fb6612dbc4 Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Fri, 9 Feb 2024 12:25:12 -0600 Subject: [PATCH 117/366] Adding M PE layout for chrysalis --- cime_config/allactive/config_pesall.xml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 3392ad71ca1..53e1dd2e337 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -843,6 +843,29 @@ + + + + -compset WCYCL*/CRYO* -res SOwISC12to30E3r3* on 52 nodes pure-MPI, ~8.5 sypd + + 1350 + 128 + 128 + 1280 + 1920 + 1408 + + + 0 + 1280 + 1280 + 0 + 1408 + 0 + + + + From d47632e624dad3cfab3c01752e271b6668cf33d6 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 12 Jul 2024 01:27:16 -0500 Subject: [PATCH 118/366] Update file datestamps --- components/mpas-ocean/cime_config/buildnml | 8 ++++---- components/mpas-seaice/cime_config/buildnml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 819572f617c..acc0d2b64b0 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -407,17 +407,17 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' elif ocn_grid == 'SOwISC12to30E3r3': - decomp_date = '20240705' + decomp_date = '20240718' decomp_prefix = 'partitions/mpas-o.graph.info.' - restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240705.nc' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240718.nc' analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' - ic_date = '20240705' + ic_date = '20240718' ic_prefix = 'mpaso.SOwISC12to30E3r3' if ocn_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") logger.warning(" But no file available for this grid.") if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240705.n' + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240718.n' #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index 821b2d09d54..a53ee01324c 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -331,11 +331,11 @@ def buildnml(case, caseroot, compname): logger.warning(" But no file available for this grid.") elif ice_grid == 'SOwISC12to30E3r3': - grid_date = '20240705' + grid_date = '20240718' grid_prefix = 'mpassi.SOwISC12to30E3r3' - decomp_date = '20240705' + decomp_date = '20240718' decomp_prefix = 'partitions/mpas-seaice.graph.info.' - data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240705.nc' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240718.nc' if ice_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") From eb834f151fb1e13ee8da680136c230f7873d74bc Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 17 Jul 2024 13:33:10 -0500 Subject: [PATCH 119/366] Add mapping and domain files for SOwISC12to30E3r3 --- cime_config/config_grids.xml | 101 ++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 0f1c3512ec8..6c70e7e9ba3 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -416,6 +416,16 @@ RRSwISC6to18E3r5 + + T62 + T62 + SOwISC12to30E3r3 + rx1 + null + null + SOwISC12to30E3r3 + + TL319 TL319 @@ -656,6 +666,16 @@ RRSwISC6to18E3r5 + + TL319 + TL319 + SOwISC12to30E3r3 + JRA025 + null + null + SOwISC12to30E3r3 + + TL319 TL319 @@ -1390,6 +1410,16 @@ RRSwISC6to18E3r5 + + ne30np4.pg2 + ne30np4.pg2 + SOwISC12to30E3r3 + r05 + null + null + SOwISC12to30E3r3 + + ne0np4_northamericax4v1 r0125 @@ -2427,6 +2457,16 @@ RRSwISC6to18E3r5 + + ne30np4.pg2 + r05 + SOwISC12to30E3r3 + r05 + null + null + SOwISC12to30E3r3 + + ne30np4.pg2 r05 @@ -2738,6 +2778,7 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.T62_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240716.nc T62 is Gaussian grid: @@ -2796,6 +2837,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240716.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_oRRS18to6v3.220124.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_oRRS18to6v3.220124.nc TL319 is JRA lat/lon grid: @@ -2909,6 +2952,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240716.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_gx1v6.190806.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_gx1v6.190806.nc ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: @@ -3227,6 +3272,13 @@ RRSwISC6to18E3r5 is a MPAS ocean grid generated with the jigsaw/compass process using a mesh density function that is roughly proportional to the Rossby radius of deformation, with 18 km gridcells at low and 6 km gridcells at high latitudes. Additionally, it has ocean in ice-shelf cavities: + + 807479 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240716.nc + SOwISC12to30E3r3 is a MPAS ocean grid generated with the jigsaw/compass process using XXXXX. Additionally, it has ocean in ice-shelf cavities: + + @@ -3263,6 +3315,8 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240716.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_gx1v6.191014.nc r05 is 1/2 degree river routing grid: @@ -3777,6 +3831,16 @@ cpl/gridmaps/ne30pg2/map_ne30pg2_to_RRSwISC6to18E3r5_trfvnp2.20240328.nc + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240716.nc + + cpl/gridmaps/ne30pg3/map_ne30pg3_to_oEC60to30v3_mono.200331.nc cpl/gridmaps/ne30pg3/map_ne30pg3_to_oEC60to30v3_bilin.200331.nc @@ -4565,6 +4629,14 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_T62_traave.20240328.nc + + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240716.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240716.nc + + cpl/gridmaps/TL319/map_TL319_to_oQU240wLI_traave.20240509.nc cpl/gridmaps/TL319/map_TL319_to_oQU240wLI-nomask_trbilin.20240509.nc @@ -4717,6 +4789,14 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_TL319_traave.20240328.nc + + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240716.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240716.nc + + cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_aave.220124.nc cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_bilin.220124.nc @@ -5101,6 +5181,10 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_r05_traave.20240328.nc + + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240716.nc + + cpl/cpl6/map_EC30to60E2r2_to_r05_neareststod.220728.nc @@ -5216,6 +5300,11 @@ cpl/cpl6/map_rx1_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + + cpl/cpl6/map_JRA025_to_oQU240wLI_cstmnn.r150e300.20240516.nc cpl/cpl6/map_JRA025_to_oQU240wLI_cstmnn.r150e300.20240516.nc @@ -5311,6 +5400,11 @@ cpl/cpl6/map_JRA025_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + + cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc @@ -5402,10 +5496,15 @@ - cpl/cpl6/map_r05_to_RRSwISC6to18E3r5.cstmnn.r250e1250_58NS.20240328.nc + cpl/cpl6/map_r05_to_RRSwISC6to18E3r5_cstmnn.r250e1250_58NS.20240328.nc cpl/cpl6/map_r05_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + + cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc From 1f71f562a658709723794af1859c76e6971fe2e8 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 24 Jul 2024 15:47:58 -0500 Subject: [PATCH 120/366] Update mapping and domain files for the SOwISC12to30E3r3 mesh --- cime_config/config_grids.xml | 66 ++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 6c70e7e9ba3..8eb10bd2ae3 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -2778,7 +2778,7 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.T62_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240723.nc T62 is Gaussian grid: @@ -2837,8 +2837,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240716.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240723.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_oRRS18to6v3.220124.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_oRRS18to6v3.220124.nc TL319 is JRA lat/lon grid: @@ -2952,8 +2952,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240716.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240723.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_gx1v6.190806.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_gx1v6.190806.nc ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: @@ -3273,9 +3273,9 @@ - 807479 + 807630 1 - $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240723.nc SOwISC12to30E3r3 is a MPAS ocean grid generated with the jigsaw/compass process using XXXXX. Additionally, it has ocean in ice-shelf cavities: @@ -3315,8 +3315,8 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240716.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240723.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_gx1v6.191014.nc r05 is 1/2 degree river routing grid: @@ -3832,13 +3832,13 @@ - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240716.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240716.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240716.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240716.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240723.nc @@ -4630,11 +4630,11 @@ - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240716.nc - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240716.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240723.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240723.nc @@ -4790,11 +4790,11 @@ - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240716.nc - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240716.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240723.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240723.nc @@ -5182,7 +5182,7 @@ - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240723.nc @@ -5301,8 +5301,8 @@ - cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc - cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc @@ -5401,8 +5401,8 @@ - cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc - cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc @@ -5501,8 +5501,8 @@ - cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc - cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc From 2c54f55f81fce411cb71765f360ea191e1fe226d Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 25 Jul 2024 01:49:10 -0500 Subject: [PATCH 121/366] Update datestamps (again) --- components/mpas-ocean/cime_config/buildnml | 8 ++++---- components/mpas-seaice/cime_config/buildnml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index acc0d2b64b0..57913c2cab2 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -407,17 +407,17 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' elif ocn_grid == 'SOwISC12to30E3r3': - decomp_date = '20240718' + decomp_date = '20240801' decomp_prefix = 'partitions/mpas-o.graph.info.' - restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240718.nc' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240801.nc' analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' - ic_date = '20240718' + ic_date = '20240801' ic_prefix = 'mpaso.SOwISC12to30E3r3' if ocn_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") logger.warning(" But no file available for this grid.") if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240718.n' + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240801.n' #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index a53ee01324c..ae91dea2250 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -331,11 +331,11 @@ def buildnml(case, caseroot, compname): logger.warning(" But no file available for this grid.") elif ice_grid == 'SOwISC12to30E3r3': - grid_date = '20240718' + grid_date = '20240801' grid_prefix = 'mpassi.SOwISC12to30E3r3' - decomp_date = '20240718' + decomp_date = '20240801' decomp_prefix = 'partitions/mpas-seaice.graph.info.' - data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240718.nc' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240801.nc' if ice_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") From 7aa9e1d4e7ae0488ac9d944b868a06076e43ebe5 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 8 Aug 2024 13:51:56 -0500 Subject: [PATCH 122/366] Update domain and mapping file datestamps (also again) --- cime_config/config_grids.xml | 64 ++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 8eb10bd2ae3..b92fe082d7e 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -2778,7 +2778,7 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.T62_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240808.nc T62 is Gaussian grid: @@ -2837,8 +2837,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240723.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_oRRS18to6v3.220124.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_oRRS18to6v3.220124.nc TL319 is JRA lat/lon grid: @@ -2952,8 +2952,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240723.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_gx1v6.190806.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_gx1v6.190806.nc ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: @@ -3275,7 +3275,7 @@ 807630 1 - $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240808.nc SOwISC12to30E3r3 is a MPAS ocean grid generated with the jigsaw/compass process using XXXXX. Additionally, it has ocean in ice-shelf cavities: @@ -3315,8 +3315,8 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240723.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_gx1v6.191014.nc r05 is 1/2 degree river routing grid: @@ -3832,13 +3832,13 @@ - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240723.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240723.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240723.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240723.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240808.nc @@ -4630,11 +4630,11 @@ - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240723.nc - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240723.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240808.nc @@ -4790,11 +4790,11 @@ - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240723.nc - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240723.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240808.nc @@ -5182,7 +5182,7 @@ - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240808.nc @@ -5301,8 +5301,8 @@ - cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc - cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc @@ -5401,8 +5401,8 @@ - cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc - cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc @@ -5501,8 +5501,8 @@ - cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc - cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc From a3bf76465a445097a0e816a0a24790e9038ac914 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 29 Aug 2024 08:43:18 -0500 Subject: [PATCH 123/366] Update ocn and ice datestamps (again, again) --- components/mpas-ocean/cime_config/buildnml | 8 ++++---- components/mpas-seaice/cime_config/buildnml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 57913c2cab2..d1385708619 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -407,17 +407,17 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' elif ocn_grid == 'SOwISC12to30E3r3': - decomp_date = '20240801' + decomp_date = '20240829' decomp_prefix = 'partitions/mpas-o.graph.info.' - restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240801.nc' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240829.nc' analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' - ic_date = '20240801' + ic_date = '20240829' ic_prefix = 'mpaso.SOwISC12to30E3r3' if ocn_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") logger.warning(" But no file available for this grid.") if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240801.n' + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240829.n' #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index ae91dea2250..2fee2d12b11 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -331,11 +331,11 @@ def buildnml(case, caseroot, compname): logger.warning(" But no file available for this grid.") elif ice_grid == 'SOwISC12to30E3r3': - grid_date = '20240801' + grid_date = '20240829' grid_prefix = 'mpassi.SOwISC12to30E3r3' - decomp_date = '20240801' + decomp_date = '20240829' decomp_prefix = 'partitions/mpas-seaice.graph.info.' - data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240801.nc' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240829.nc' if ice_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") From 327638728f727c19c795b51ed8a8477d66a3b252 Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Mon, 16 Sep 2024 10:57:13 -0500 Subject: [PATCH 124/366] Adding ocn/ice ICs for B-cases, fixing typo in dismf file extention --- components/mpas-ocean/cime_config/buildnml | 6 +++--- components/mpas-seaice/cime_config/buildnml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index d1385708619..e4b2f7f8b42 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -414,10 +414,10 @@ def buildnml(case, caseroot, compname): ic_date = '20240829' ic_prefix = 'mpaso.SOwISC12to30E3r3' if ocn_ic_mode == 'spunup': - logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") - logger.warning(" But no file available for this grid.") + ic_date = '20240829' + ic_prefix = 'mpaso.SOwISC12to30E3r3.rstFromG-chrysalis' if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240829.n' + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240829.nc' #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index 2fee2d12b11..471fbf9233f 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -337,8 +337,8 @@ def buildnml(case, caseroot, compname): decomp_prefix = 'partitions/mpas-seaice.graph.info.' data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240829.nc' if ice_ic_mode == 'spunup': - logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") - logger.warning(" But no file available for this grid.") + grid_date = '20240829' + grid_prefix = 'mpassi.SOwISC12to30E3r3.rstFromG-chrysalis' elif ice_grid == 'ICOS10': grid_date = '211015' From 5a5e7f9c93e34059db8ea1f8007cf97886b30672 Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Fri, 27 Sep 2024 12:15:07 -0500 Subject: [PATCH 125/366] Adding default finidat for 1850 conditions with r05 land grid --- components/elm/bld/namelist_files/namelist_defaults.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index beeae785859..cf3c7ef0b09 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -232,6 +232,13 @@ ic_tod="0" sim_year="1850" glc_nec="0" use_crop=".false." >lnd/clm2/initdata_map ic_tod="0" sim_year="2000" glc_nec="0" use_crop=".false." >lnd/clm2/initdata_map/clmi.ICRUCLM45SP.2000-01-01.ne240np4_tx0.1v2.162cd32_simyr2000_c170910.nc + + +lnd/clm2/initdata/elmi.v3-SORRM.ne30pg2_r05_SOwISC12to30E3r3.1850-01-01-00000.c20240923.nc + + + lnd/clm2/surfdata_map/surfdata_ne0np4_northamericax4v1.pg2_simyr1850_c210112_with_TOP.nc From d6bf9943ca314a693e2db46d5235e4c024ded28e Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 28 Sep 2024 15:38:48 -0400 Subject: [PATCH 126/366] try new slimmed container --- .github/workflows/eamxx-gh-ci-standalone.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index 4a194a89106..f916384476a 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -25,8 +25,10 @@ jobs: test: - sp - opt + - dbg + - cov container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.2 + image: ghcr.io/e3sm-project/containers-standalone-ghci:standalone-ghci-0.1.0 steps: - From da9951ac22463f0443b47daf3445d4a723a797e1 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 28 Sep 2024 16:15:11 -0400 Subject: [PATCH 127/366] don't run coverage test --- .github/workflows/eamxx-gh-ci-standalone.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index f916384476a..ffb9afb02fe 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -26,7 +26,6 @@ jobs: - sp - opt - dbg - - cov container: image: ghcr.io/e3sm-project/containers-standalone-ghci:standalone-ghci-0.1.0 From 0012b49af6326a8213b2f8b55d586e69856b1b5a Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 28 Sep 2024 18:42:11 -0500 Subject: [PATCH 128/366] add fpe test --- .github/workflows/eamxx-gh-ci-standalone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index ffb9afb02fe..9fde89ed35b 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -26,6 +26,7 @@ jobs: - sp - opt - dbg + - fpe container: image: ghcr.io/e3sm-project/containers-standalone-ghci:standalone-ghci-0.1.0 From b33da8c83090b4c7a468f5f04caa8c63e165e55c Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 28 Sep 2024 18:45:44 -0500 Subject: [PATCH 129/366] update cosp2 submod --- .gitmodules | 4 ++-- components/eam/src/physics/cosp2/external | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 69d56e14036..b36d90a00a4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,8 +22,8 @@ branch = scorpio_classic [submodule "cosp2"] path = components/eam/src/physics/cosp2/external - url = git@github.com:CFMIP/COSPv2.0.git - branch = CESM_v2.1.4 + url = git@github.com:bartgol/COSPv2.0.git + branch = bartgol/fix-cosp_optical_inputs [submodule "cime"] path = cime url = git@github.com:ESMCI/cime.git diff --git a/components/eam/src/physics/cosp2/external b/components/eam/src/physics/cosp2/external index 9d910acba3e..2deb41975fa 160000 --- a/components/eam/src/physics/cosp2/external +++ b/components/eam/src/physics/cosp2/external @@ -1 +1 @@ -Subproject commit 9d910acba3e3a3151de231184d4b109f65e28aee +Subproject commit 2deb41975faa4f5eacfc8b8f12b85acf6583d407 From 632faeb0ff03a1f012b658de68d4b11f7c6ced45 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Mon, 30 Sep 2024 14:53:49 -0500 Subject: [PATCH 130/366] Use same balance_check_tolerance for C and P --- components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index a2ad41bf665..1a9b59f776e 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -281,7 +281,7 @@ subroutine ColCBalanceCheck(bounds, & end if ! check for significant errors - if (abs(col_errcb(c)) > 1e-8_r8) then + if (abs(col_errcb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if @@ -734,7 +734,7 @@ subroutine ColPBalanceCheck(bounds, & col_errpb(c) = (col_pinputs(c) - col_poutputs(c))*dt - & (col_endpb(c) - col_begpb(c)) - if (abs(col_errpb(c)) > 1e-8_r8) then + if (abs(col_errpb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if From f8dfc6e3eaef99d5fefe0f1af234e4e71969887a Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 1 Oct 2024 09:22:15 -0500 Subject: [PATCH 131/366] add rtd to mesh file name --- components/ww3/bld/build-namelist | 2 +- components/ww3/cime_config/buildnml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/ww3/bld/build-namelist b/components/ww3/bld/build-namelist index 915e6b0bf2a..092dab3ffba 100755 --- a/components/ww3/bld/build-namelist +++ b/components/ww3/bld/build-namelist @@ -363,7 +363,7 @@ if ($NML_TYPE eq "ww3_grid") { add_default($nl, 'grid%dmin'); add_default($nl, 'unst%sf'); - add_default($nl, 'unst%filename', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/${WAV_GRID}.msh'"); + add_default($nl, 'unst%filename', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/${WAV_GRID}_rtd.msh'"); add_default($nl, 'unst%idf'); add_default($nl, 'unst%idla'); add_default($nl, 'unst%idfm'); diff --git a/components/ww3/cime_config/buildnml b/components/ww3/cime_config/buildnml index f550d032188..3fe5d79d3f8 100755 --- a/components/ww3/cime_config/buildnml +++ b/components/ww3/cime_config/buildnml @@ -64,7 +64,7 @@ def buildnml(case, caseroot, compname): with open(os.path.join(casebuild, "ww3.input_data_list"), "w") as input_list: - input_list.write("mesh = {}/wav/ww3/{}.msh\n".format(din_loc_root,wav_grid)) + input_list.write("mesh = {}/wav/ww3/{}_rtd.msh\n".format(din_loc_root,wav_grid)) input_list.write("stations = {}/wav/ww3/stations.txt\n".format(din_loc_root)) input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) From e482b69a80619032d0ed44e02224c323df3a3184 Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Tue, 1 Oct 2024 12:26:14 -0400 Subject: [PATCH 132/366] adjusted 1. to 1._r8 in VerticalProfileMod --- .../elm/src/biogeochem/VerticalProfileMod.F90 | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index d5bd1d1c83f..3d315b95eb6 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -212,10 +212,10 @@ subroutine decomp_vertprofiles(bounds, & end do else ! if fully frozen, or no roots, put everything in the top layer - froot_prof(p,1) = 1./dzsoi_decomp(1) - croot_prof(p,1) = 1./dzsoi_decomp(1) - leaf_prof(p,1) = 1./dzsoi_decomp(1) - stem_prof(p,1) = 1./dzsoi_decomp(1) + froot_prof(p,1) = 1._r8/dzsoi_decomp(1) + croot_prof(p,1) = 1._r8/dzsoi_decomp(1) + leaf_prof(p,1) = 1._r8/dzsoi_decomp(1) + stem_prof(p,1) = 1._r8/dzsoi_decomp(1) endif end do @@ -257,9 +257,9 @@ subroutine decomp_vertprofiles(bounds, & pdep_prof(c,j) = surface_prof(j)/ surface_prof_tot end do else - nfixation_prof(c,1) = 1./dzsoi_decomp(1) - ndep_prof(c,1) = 1./dzsoi_decomp(1) - pdep_prof(c,1) = 1./dzsoi_decomp(1) + nfixation_prof(c,1) = 1._r8/dzsoi_decomp(1) + ndep_prof(c,1) = 1._r8/dzsoi_decomp(1) + pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif else if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then @@ -271,9 +271,9 @@ subroutine decomp_vertprofiles(bounds, & end if end do else - nfixation_prof(c,1) = 1./dzsoi_decomp(1) - ndep_prof(c,1) = 1./dzsoi_decomp(1) - pdep_prof(c,1) = 1./dzsoi_decomp(1) + nfixation_prof(c,1) = 1._r8/dzsoi_decomp(1) + ndep_prof(c,1) = 1._r8/dzsoi_decomp(1) + pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif end if end do From bbacf3a8f16f33091880bc1850b0e4a9cbecca35 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 1 Oct 2024 17:19:33 -0500 Subject: [PATCH 133/366] Update README and other files for v3.0.1 Update README and other files for v3.0.1 --- CITATION.cff | 2 +- LICENSE | 16 +++++++++++----- README.md | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 9542a7d01ef..78099055fc3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ authors: - given-names: E3SM family-names: Project version: 3.0.0 -doi: 10.11578/E3SM/dc.20240301.3 +doi: 10.11578/E3SM/dc.20240930.1 repository-code: 'https://github.com/E3SM-Project/E3SM' url: 'https://e3sm.org' license: BSD-3-Clause diff --git a/LICENSE b/LICENSE index d74a2aa127a..e5e69b7c906 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,7 @@ Except for the separable pieces descibed below, E3SM is released under the following 3-Clause BSD Open Source license. ******************************************************************************* -Copyright ©2023, UChicago Argonne, LLC All Rights Reserved +Copyright ©2043, UChicago Argonne, LLC All Rights Reserved Software Name: Energy Exascale Earth System Model (E3SM) @@ -55,15 +55,21 @@ GPTL share/timing author non-commeric MCT externals/mct ANL BSD YAKL externals/YAKL author BSD cub externals/cub author, NVIDIA BSD -kokkos externals/kokkos SNL BSD +kokkos externals/ekat SNL BSD +haero externals/haero SNL/Battelle BSD +mam4xx externals/mam4xx SNL/Battelle BSD Ocean/Ice under components/ ----------- ----------------- CICE cice LANL BSD MPAS Framework mpas-framework LANL BSD -MPAS Ocean mpas-ocean LANL BSD -MPAS SeaIce mpas-seaice LANL BSD -MPAS-Albany LandIce mpas-albany-landice LANL, SNL BSD +FFTW mpas-ocean/src/FFTW author/MIT GPL +MARBL mpas-ocean/src/MARBL NCAR BSD +SHTNS mpas-ocean/src/SHTNS CeCILL GPL + +Land-ice under components/mpas-albany-landice +----------- ------------------------------------ +SeaLevelModel src/SeaLevelModel author MIT Land under components/elm/src ----------- ------------------------ diff --git a/README.md b/README.md index 192c288d78e..1601f9cb903 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ the most challenging and demanding climate-change research problems and Department of Energy mission needs while efficiently using DOE Leadership Computing Facilities. -DOI: [10.11578/E3SM/dc.20240301.3](http://dx.doi.org/10.11578/E3SM/dc.20240301.3) +DOI: [10.11578/E3SM/dc.20240930.1](http://dx.doi.org/10.11578/E3SM/dc.20240930.1) Please visit the [project website](https://e3sm.org) or our [Confluence site](https://acme-climate.atlassian.net/wiki/spaces/DOC/overview) for further details. For questions about the model, use [Github Discussions](https://github.com/E3SM-Project/E3SM/discussions). -See our Github-hosted documentation at [https://e3sm-project.github.io/E3SM/](https://e3sm-project.github.io/E3SM/). +See our Github-hosted documentation at [https://docs.e3sm.org/E3SM](https://docs.e3sm.org/E3SM/). Table of Contents -------------------------------------------------------------------------------- From d5d8f6a8f1e00e5848b12e28aff10457dbf48548 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 1 Oct 2024 17:55:02 -0500 Subject: [PATCH 134/366] Disable linting on README.md Disable linting on README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1601f9cb903..13c927b8c7f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + [![E3SM Logo](https://e3sm.org/wp-content/themes/e3sm/assets/images/e3sm-logo.png)](https://e3sm.org) Energy Exascale Earth System Model (E3SM) From 00377667bab08e27ff7fccec82d94984dfab04e9 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 1 Oct 2024 18:23:43 -0500 Subject: [PATCH 135/366] Mention more packages in LICENSE Mention more packages in LICENSE --- LICENSE | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index e5e69b7c906..ba380e46531 100644 --- a/LICENSE +++ b/LICENSE @@ -66,6 +66,14 @@ MPAS Framework mpas-framework LANL BSD FFTW mpas-ocean/src/FFTW author/MIT GPL MARBL mpas-ocean/src/MARBL NCAR BSD SHTNS mpas-ocean/src/SHTNS CeCILL GPL +cvmix mpas-ocean/src/cvmix NCAR LGPL +gotm mpas-ocean/src/gotm authors GPL +ppr mpas-ocean/src/ppr author custom +Icepack mpas-seaice/src/icepack LANL BSD + +Waves under components/ww3 +----------- -------------------- +WW3 src/WW3 NWS/NOAA custom Land-ice under components/mpas-albany-landice ----------- ------------------------------------ @@ -97,7 +105,7 @@ HOMMEXX components/homme/src/share/cxx SNL BSD Actual copyright holder for above Institutions: NCAR = University Corporation for Atmospheric Research -LANL = Los Alamos National Security, LLC +LANL = Los Alamos National Security, LLC, Triad National Security, LLC SNL = National Technology & Engineering Solutions of Sandia, LLC LBNL = The Regents of the University of California, through Lawrence Berkeley National Laboratory From d866026295488f8f657e42b8d4c487568e590078 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 2 Oct 2024 15:11:02 -0500 Subject: [PATCH 136/366] Add sed commands to modify env_archive.xml to test the conservation am --- .../testmods_dirs/mpaso/conservation_check/shell_commands | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands new file mode 100644 index 00000000000..764340d699f --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands @@ -0,0 +1,5 @@ +# include mpas-ocean outputs in testing +sed -i 's/\(compname="mpaso" exclude_testing="true"\)/compname="mpaso"/' env_archive.xml +sed -i '/\(compname="mpaso"\)/,// s/hist/hist.am.conservationCheck\\..*\\.nc$/' env_archive.xml +sed -i 's/casename.mpaso.hist.am.globalStats.1976-01-01.nc/casename.mpaso.hist.am.conservationCheck.1976-01-01.nc/' env_archive.xml +sed -i '/casename.mpaso.hist.am.highFrequencyOutput.1976-01-01_00.00.00.nc/d' env_archive.xml From 55f473dcdec35a66b9b7dfa80ca70bea00a22b47 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Wed, 2 Oct 2024 16:03:28 -0500 Subject: [PATCH 137/366] Fix typo in LICENSE Fix typo --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index ba380e46531..247287ab4c0 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,7 @@ Except for the separable pieces descibed below, E3SM is released under the following 3-Clause BSD Open Source license. ******************************************************************************* -Copyright ©2043, UChicago Argonne, LLC All Rights Reserved +Copyright 2024, UChicago Argonne, LLC All Rights Reserved Software Name: Energy Exascale Earth System Model (E3SM) From 16e80809262ebc2652ada3ed43737c174f9d90b3 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 3 Oct 2024 14:38:05 -0500 Subject: [PATCH 138/366] Change GMPAS-JRA1p5-DIB-PISMF-DSGR test from SMS to PET to check omp --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b8397..f23cea24012 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -275,7 +275,7 @@ "ERS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "PEM_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-TMIX.mpaso-jra_1958", - "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-DSGR.mpaso-jra_1958", + "PET_P480_Ld2.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-DSGR.mpaso-jra_1958", ) }, From aba69664dad83c9624b28173daed6cb90d491ce6 Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Thu, 3 Oct 2024 17:51:59 -0400 Subject: [PATCH 139/366] Added debugging info --- components/elm/src/biogeochem/VerticalProfileMod.F90 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index 3d315b95eb6..a1b84fe945f 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -336,7 +336,15 @@ subroutine decomp_vertprofiles(bounds, & end do if ( ( abs(froot_prof_sum - 1._r8) > delta ) .or. ( abs(croot_prof_sum - 1._r8) > delta ) .or. & ( abs(stem_prof_sum - 1._r8) > delta ) .or. ( abs(leaf_prof_sum - 1._r8) > delta ) ) then + c = veg_pp%column(p) write(iulog, *) 'profile sums: ', froot_prof_sum, croot_prof_sum, leaf_prof_sum, stem_prof_sum + write(iulog, *) 'c: ',c + write(iulog, *) 'altmax_lastyear_indx: ', altmax_lastyear_indx(c) + write(iulog, *) 'cinput_rootfr: ', col_cinput_rootfr(c,:) + write(iulog, *) 'dzsoi_decomp: ', dzsoi_decomp(:) + write(iulog, *) 'surface_prof: ', surface_prof(:) + write(iulog, *) 'p, itype(p), wtcol(p): ', p, veg_pp%itype(p), veg_pp%wtcol(p) + write(iulog, *) 'cinput_rootfr(p,:): ', cinput_rootfr(p,:) call endrun(msg=' ERROR: sum-1 > delta'//errMsg(__FILE__, __LINE__)) endif end do From cc1ff16763306a34335a13299f78f632b2b37248 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 3 Oct 2024 20:36:35 -0500 Subject: [PATCH 140/366] print out moab counter only with MOABDEBUG --- driver-moab/main/cime_comp_mod.F90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/driver-moab/main/cime_comp_mod.F90 b/driver-moab/main/cime_comp_mod.F90 index 265f8e11338..59fa2daf408 100644 --- a/driver-moab/main/cime_comp_mod.F90 +++ b/driver-moab/main/cime_comp_mod.F90 @@ -2896,10 +2896,12 @@ subroutine cime_run() num_moab_exports = num_moab_exports + 1! this is moab clock used for debugging call seq_timemgr_clockAdvance( seq_SyncClock, force_stop, force_stop_ymd, force_stop_tod) call seq_timemgr_EClockGetData(EClock_d, stepno=cur_step_no) +#ifdef MOABDEBUG if (iamroot_CPLID) then write(logunit,*) ' num_moab_exports , cur_step_no ',num_moab_exports, cur_step_no call shr_sys_flush(logunit) endif +#endif call seq_timemgr_EClockGetData( EClock_d, curr_ymd=ymd, curr_tod=tod) call shr_cal_date2ymd(ymd,year,month,day) stop_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_stop) From 60c99c36012bbb5c8e3bc0ab2a2e7ac68ad73501 Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Sun, 6 Oct 2024 11:08:42 -0500 Subject: [PATCH 141/366] Tweaking Chrysalis PE layout --- cime_config/allactive/config_pesall.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 53e1dd2e337..96ca63f5fbe 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -848,17 +848,17 @@ -compset WCYCL*/CRYO* -res SOwISC12to30E3r3* on 52 nodes pure-MPI, ~8.5 sypd - 1350 - 128 - 128 - 1280 + 1408 + 384 + 384 + 1024 1920 1408 0 - 1280 - 1280 + 1024 + 1024 0 1408 0 From 2481ef6af48f0eecbd0c800e927166e092257d7a Mon Sep 17 00:00:00 2001 From: Michael Kelleher Date: Mon, 7 Oct 2024 10:01:24 -0500 Subject: [PATCH 142/366] Fix import of moved method --- components/mpas-ocean/cime_config/SystemTests/mvko.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/SystemTests/mvko.py b/components/mpas-ocean/cime_config/SystemTests/mvko.py index 71477a4dfc1..90fa4b7a553 100644 --- a/components/mpas-ocean/cime_config/SystemTests/mvko.py +++ b/components/mpas-ocean/cime_config/SystemTests/mvko.py @@ -18,6 +18,7 @@ import netCDF4 as nc import CIME.test_status +from CIME.status import append_testlog import CIME.utils from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup @@ -394,4 +395,4 @@ def _compare_baseline(self): f" {viewing}" ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) From f5b8ad73403b8390a9c587ea67c201a95925d97c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 7 Oct 2024 11:38:53 -0600 Subject: [PATCH 143/366] Adds NOOPT_FILES concept to mpas cmake system And uses this system to deoptimize icepack_shortwave_data.F90, a file that takes a very long time to compile and is just setting up consts. --- components/mpas-framework/src/build_core.cmake | 7 +++++++ components/mpas-seaice/src/seaice.cmake | 1 + 2 files changed, 8 insertions(+) diff --git a/components/mpas-framework/src/build_core.cmake b/components/mpas-framework/src/build_core.cmake index 23c1935a8ab..a2c361e069e 100644 --- a/components/mpas-framework/src/build_core.cmake +++ b/components/mpas-framework/src/build_core.cmake @@ -62,6 +62,13 @@ function(build_core CORE) endforeach() endif() + # Disable optimizations on some files that would take too long to compile, expect these to all be fortran files + foreach (SOURCE_FILE IN LISTS NOOPT_FILES) + get_filename_component(SOURCE_EXT ${SOURCE_FILE} EXT) + string(REPLACE "${SOURCE_EXT}" ".f90" SOURCE_F90 ${SOURCE_FILE}) + e3sm_deoptimize_file(${CMAKE_BINARY_DIR}/${SOURCE_F90}) + endforeach() + genf90_targets("${RAW_SOURCES}" "${INCLUDES}" "${CPPDEFS}" "${NO_PREPROCESS}" "${INC_DIR}") target_sources(${COMPONENT} PRIVATE ${SOURCES}) diff --git a/components/mpas-seaice/src/seaice.cmake b/components/mpas-seaice/src/seaice.cmake index 8b72de8ec7d..5d314fd7594 100644 --- a/components/mpas-seaice/src/seaice.cmake +++ b/components/mpas-seaice/src/seaice.cmake @@ -144,6 +144,7 @@ set(SEAICE_MODEL_FORWARD ) list(APPEND RAW_SOURCES ${SEAICE_MODEL_FORWARD}) list(APPEND DISABLE_QSMP ${SEAICE_MODEL_FORWARD}) +list(APPEND NOOPT_FILES "core_seaice/icepack/columnphysics/icepack_shortwave_data.F90") # Generate core input handle_st_nl_gen("namelist.seaice" "streams.seaice stream_list.seaice. listed" ${CORE_INPUT_DIR} ${CORE_BLDDIR}) From 664c784092453530bfa63b988815fbba0d2153c6 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 7 Oct 2024 14:59:51 -0600 Subject: [PATCH 144/366] Initial step. Get it building with latest rrtmgp subm --- components/eam/src/physics/rrtmgp/external | 2 +- components/eamxx/CMakeLists.txt | 4 ++-- .../eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp | 7 ++----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 8ff525eeed1..b851a86aa93 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 8ff525eeed1d87a2ca6f251c4d16b46222c5554d +Subproject commit b851a86aa9312193f8f292f612e40afb056f90cf diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 07c3cda7672..5a7149234a5 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -212,8 +212,8 @@ endif() # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) -option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE) -option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE) +option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" FALSE) +option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" TRUE) if (SCREAM_RRTMGP_ENABLE_YAKL) add_definitions("-DRRTMGP_ENABLE_YAKL") endif() diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 216722b3766..55e6d87e1fe 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -167,9 +167,6 @@ using MDRP = typename conv::MDRP; template using view_t = Kokkos::View; -template -using oview_t = Kokkos::Experimental::OffsetView; - template using hview_t = Kokkos::View; @@ -723,7 +720,7 @@ static void rrtmgp_sw( val |= p_lay(0, 0) < p_lay(0, nlay-1); }, Kokkos::LOr(top_at_1)); - oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + view_t col_gas("col_gas", ncol, nlay, k_dist.get_ngas()+1); k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); if (extra_clnsky_diag) { @@ -907,7 +904,7 @@ static void rrtmgp_lw( limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); // Do gas optics - oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + view_t col_gas("col_gas", ncol, nlay, k_dist.get_ngas()+1); k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); if (extra_clnsky_diag) { k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); From e1dff58884c2114e76c953b2460e03a2092447d1 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 7 Oct 2024 15:37:53 -0700 Subject: [PATCH 145/366] use FI_MR_CACHE_MONITOR=kdreg2 for all nersc machines --- cime_config/machines/config_machines.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 140b733f83d..76c206ed0b2 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -278,6 +278,7 @@ FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -444,6 +445,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -587,6 +589,7 @@ FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software + kdreg2 MPI_Bcast $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} @@ -756,6 +759,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -898,6 +902,7 @@ FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} From f37edf7fdc23b23b01dcf4f73d62f79a44a6d92f Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 7 Oct 2024 16:31:11 -0700 Subject: [PATCH 146/366] correct default fates parameter for API36.1 --- components/elm/bld/namelist_files/namelist_defaults.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 7d2390d267b..896fe5d7396 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -134,7 +134,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.36.1.0_12pft_c240926.nc +lnd/clm2/paramdata/fates_params_api.36.1.0_14pft_c241003.nc lnd/clm2/paramdata/CNP_parameters_c131108.nc From 39ca74583f72e8fb9286954c10ded5c8ba994fd1 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 7 Oct 2024 16:36:39 -0700 Subject: [PATCH 147/366] update fates submodule to sci.1.78.3_api.36.1.0 --- components/elm/src/external_models/fates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/external_models/fates b/components/elm/src/external_models/fates index 1982b0032c3..825579d0b40 160000 --- a/components/elm/src/external_models/fates +++ b/components/elm/src/external_models/fates @@ -1 +1 @@ -Subproject commit 1982b0032c3cab6278892eccb85f643114ffb1af +Subproject commit 825579d0b406fe99344591b5ed8356e5c7aeebec From ed346d4bb74a67593308a85e8a7dd8ab5ffec564 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 7 Oct 2024 16:39:45 -0700 Subject: [PATCH 148/366] update fates hydro test module to change sapwood allometry mode This update calls the fates parameter modification tooling to set the grass allometry smode to 1, which is the default for all other fates PFTs. This is a temporary workaround for NGEET/fates#1254. --- .../testmods_dirs/elm/fateshydro/shell_commands | 15 +++++++++++++++ .../testmods_dirs/elm/fateshydro/user_nl_elm | 1 + 2 files changed, 16 insertions(+) diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands index 84648f9222c..e82b5ce85ff 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands @@ -1,2 +1,17 @@ #!/bin/bash if [ `./xmlquery --value MACH` == bebop ]; then ./xmlchange --id LND_PIO_TYPENAME --val netcdf; fi +module load e4s +spack env activate gcc +spack load nco + +SRCDIR=`./xmlquery SRCROOT --value` +CASEDIR=`./xmlquery CASEROOT --value` +FATESDIR=$SRCDIR/src/fates +FATESPARAMFILE=$CASEDIR/fates_params_hydrograsstempfix.nc + +ncgen -o $FATESPARAMFILE $FATESDIR/parameter_files/fates_params_default.cdl + +$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_allom_smode --val 1 --allpfts + +spack unload nco +module unload e4s diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm index c74b77d05dc..ea831e5483b 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm @@ -5,6 +5,7 @@ hist_nhtfrq = -24 hist_empty_htapes = .true. use_fates_planthydro = .true. fates_spitfire_mode = 1 +fates_paramfile = '$CASEROOT/fates_params_hydrograsstempfix.nc' hist_fincl1 = 'FATES_ERRH2O_SZPF', 'FATES_TRAN_SZPF', 'FATES_SAPFLOW_SZPF', 'FATES_ITERH1_SZPF','FATES_ABSROOT_H2O_SZPF', 'FATES_TRANSROOT_H2O_SZPF','FATES_STEM_H2O_SZPF','FATES_LEAF_H2O_SZPF', From 58125633d7253aca8e22fe83e56eaa6722f7eee1 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 8 Oct 2024 08:31:51 -0600 Subject: [PATCH 149/366] replaceconfig_flux_attenuation_coefficient_runoff with config_flux_attenuation_coefficient_subglacial_runoff in subglacial kpp loop --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 1777e1f6d1e..0334667f2f1 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3574,8 +3574,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & do k = maxLevelCell(iCell), minLevelCell(iCell), -1 zBot = zTop - layerThickness(k,iCell) if (k == minLevelCell(iCell)) then - transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) - transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) + transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_subglacial_runoff, -100.0_RKIND) ) + transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_subglacial_runoff, -100.0_RKIND) ) fracAbsorbedSubglacialRunoff = transmissionCoeffTop - transmissionCoeffBot end if zTop = zBot From 7ba6c85a8e907d56713aaa78a13b0966586cb9f4 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 9 Oct 2024 07:29:38 -0400 Subject: [PATCH 150/366] Address review comments --- components/elm/docs/user-guide/interpinic.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md index fcb897834f2..f868ee7e8d2 100644 --- a/components/elm/docs/user-guide/interpinic.md +++ b/components/elm/docs/user-guide/interpinic.md @@ -47,6 +47,8 @@ cd components/elm/tools/interpinic/src export USER_LDFLAGS="-L$NETCDF_C_DIR/lib -lnetcdf -L$NETCDF_F_DIR/lib -lnetcdff -L$HDF5_DIR/lib -lhdf5" USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 + +cd ../ ``` ## 4. Run `interpinic` @@ -66,10 +68,10 @@ The `interpinic` can then be run via the following batch job (e.g., `remap.r025_ # Load relevant modules. cd -eval $(./cime/CIME/Tools/get_case_env/get_case_env) +eval $(./cime/CIME/Tools/get_case_env) # Change dir to `interpinic` -cd components/elm/tools/interpinic/src +cd components/elm/tools/interpinic/ srun -n 1 ./interpinic \ -i /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc \ From 468866d8b3c603ea239eb111fb8df6216faac348 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 9 Oct 2024 11:45:49 -0500 Subject: [PATCH 151/366] Make bld files consistent with Registry --- .../bld/namelist_files/namelist_definition_mpassi.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml index efbaef07cb9..ba6ae4be4bb 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml @@ -1077,7 +1077,7 @@ Default: Defined in namelist_defaults.xml -Use the humic matter tracer +Use the humic (refractory dissolved organic matter) tracer Valid values: true or false Default: Defined in namelist_defaults.xml @@ -1949,7 +1949,7 @@ Default: Defined in namelist_defaults.xml -Transport type of humics +Transport type of humics (refractory dissolved organic matter) Valid values: -1 = entirely in the mobile phase; 0 = retention dominated; 1 = release dominated; 0.5 = equal but rapid exchange; 2 = equal but slow exchange Default: Defined in namelist_defaults.xml From 6f3edeae3bd2a56ff3fb803bbba3d5986a8090d4 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 9 Oct 2024 17:09:34 -0400 Subject: [PATCH 152/366] keep github actions working for e3sm repo --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 04f7fcb4ffc..6c9ee1ab114 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -25,7 +25,7 @@ on: jobs: ci: - if: false + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index d3971758991..852bceec535 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -13,7 +13,7 @@ on: jobs: scream-defaults: - if: false + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest outputs: event_name: ${{ github.event_name }} From 64f041b9f8db1089902c956fe293d900d78ae159 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 10 Oct 2024 11:22:10 -0700 Subject: [PATCH 153/366] Make sure SCREAM_LIBS_ONLY is on We don't want testing stuff or small kernel stuff --- components/eam/src/physics/crm/pam/CMakeLists.txt | 3 ++- components/eamxx/CMakeLists.txt | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/CMakeLists.txt b/components/eam/src/physics/crm/pam/CMakeLists.txt index 4a8f88f669b..ac629fa89fd 100644 --- a/components/eam/src/physics/crm/pam/CMakeLists.txt +++ b/components/eam/src/physics/crm/pam/CMakeLists.txt @@ -9,8 +9,9 @@ set(PAM_DRIVER_SRC params.F90) add_library(pam_driver - ${PAM_DRIVER_SRC}) + ${PAM_DRIVER_SRC}) +set(SCREAM_LIBS_ONLY TRUE) set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) add_library(eamxx_physics INTERFACE ${SCREAM_HOME}/components/eamxx/src/physics/) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 07c3cda7672..9b6f54e5b05 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -200,6 +200,9 @@ option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them") set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") +message("JGF DEFAUL_SMALL_KERNELS is ${DEFAULT_SMALL_KERNELS}") +message("JGF SCREAM_SMALL_KERNELS is ${SCREAM_SMALL_KERNELS}") +message("JGF SCREAM_P3_SMALL_KERNELS is ${SCREAM_P3_SMALL_KERNELS}") set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_SHOC_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") From 10099ac1b8d7a9a8b5ba3b49ac01e38eb30dba8d Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 10 Oct 2024 11:22:10 -0700 Subject: [PATCH 154/366] Make sure SCREAM_LIBS_ONLY is on We don't want testing stuff or small kernel stuff --- components/eam/src/physics/crm/pam/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eam/src/physics/crm/pam/CMakeLists.txt b/components/eam/src/physics/crm/pam/CMakeLists.txt index 4a8f88f669b..c5559452384 100644 --- a/components/eam/src/physics/crm/pam/CMakeLists.txt +++ b/components/eam/src/physics/crm/pam/CMakeLists.txt @@ -11,6 +11,7 @@ set(PAM_DRIVER_SRC add_library(pam_driver ${PAM_DRIVER_SRC}) +set(SCREAM_LIBS_ONLY TRUE) set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) add_library(eamxx_physics INTERFACE ${SCREAM_HOME}/components/eamxx/src/physics/) From 2cdecfdd5cebf8a67fc687f743f49cac538de67d Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Thu, 10 Oct 2024 15:44:43 -0400 Subject: [PATCH 155/366] Added more output and changed > 0._r8 checks to > tiny --- components/elm/src/biogeochem/VerticalProfileMod.F90 | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index a1b84fe945f..1fa3db093b4 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -84,6 +84,7 @@ subroutine decomp_vertprofiles(bounds, & real(r8) :: nfixation_prof_sum real(r8) :: pdep_prof_sum real(r8) :: delta = 1.e-10_r8 + real(r8), parameter :: smallparameter = tiny(1._r8) character(len=32) :: subname = 'decomp_vertprofiles' !----------------------------------------------------------------------- @@ -191,7 +192,7 @@ subroutine decomp_vertprofiles(bounds, & surface_prof_tot = surface_prof_tot + surface_prof(j) * dzsoi_decomp(j) end if end do - if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > smallparameter) .and. (surface_prof_tot > smallparameter) ) then ! where there is not permafrost extending to the surface, integrate the profiles over the active layer ! this is equivalnet to integrating over all soil layers outside of permafrost regions do j = 1, min(max(altmax_lastyear_indx(c), 1), nlevdecomp) @@ -250,7 +251,7 @@ subroutine decomp_vertprofiles(bounds, & surface_prof_tot = surface_prof_tot + surface_prof(j) * dzsoi_decomp(j) end do if(col_pp%is_fates(c))then - if ( (altmax_lastyear_indx(c) > 0) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (surface_prof_tot > smallparameter) ) then do j = 1,min(alt_ind, nlevbed) nfixation_prof(c,j) = surface_prof(j)/ surface_prof_tot ndep_prof(c,j) = surface_prof(j)/ surface_prof_tot @@ -262,7 +263,7 @@ subroutine decomp_vertprofiles(bounds, & pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif else - if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > smallparameter) .and. (surface_prof_tot > smallparameter) ) then do j = 1, min(max(altmax_lastyear_indx(c), 1), nlevdecomp) nfixation_prof(c,j) = col_cinput_rootfr(c,j) / rootfr_tot if (j <= nlevbed) then @@ -345,6 +346,10 @@ subroutine decomp_vertprofiles(bounds, & write(iulog, *) 'surface_prof: ', surface_prof(:) write(iulog, *) 'p, itype(p), wtcol(p): ', p, veg_pp%itype(p), veg_pp%wtcol(p) write(iulog, *) 'cinput_rootfr(p,:): ', cinput_rootfr(p,:) + write(iulog,*) 'croot_prof(p,:): ',croot_prof(p,:) + write(iulog,*) 'froot_prof(p,:): ',froot_prof(p,:) + write(iulog,*) 'leaf_prof(p,:): ',leaf_prof(p,:) + write(iulog,*) 'stem_prof(p,:): ',stem_prof(p,:) call endrun(msg=' ERROR: sum-1 > delta'//errMsg(__FILE__, __LINE__)) endif end do From ae89098576c75ab1796654a77de28851b8c4874c Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 10 Oct 2024 15:12:26 -0500 Subject: [PATCH 156/366] Add config_create_all_logs_in_e3sm option to control log file creation --- .../mpas-albany-landice/bld/build-namelist | 1 + .../bld/build-namelist-section | 1 + .../namelist_files/namelist_defaults_mali.xml | 1 + .../namelist_definition_mali.xml | 8 ++++ .../mpas-albany-landice/driver/glc_comp_mct.F | 38 ++++++++++--------- .../mpas-albany-landice/src/Registry.xml | 4 ++ 6 files changed, 36 insertions(+), 17 deletions(-) diff --git a/components/mpas-albany-landice/bld/build-namelist b/components/mpas-albany-landice/bld/build-namelist index 15272f7a2da..f08d94a0e27 100755 --- a/components/mpas-albany-landice/bld/build-namelist +++ b/components/mpas-albany-landice/bld/build-namelist @@ -620,6 +620,7 @@ add_default($nl, 'config_pio_stride'); add_default($nl, 'config_year_digits'); add_default($nl, 'config_output_external_velocity_solver_data'); add_default($nl, 'config_write_albany_ascii_mesh'); +add_default($nl, 'config_create_all_logs_in_e3sm'); ################################# # Namelist group: decomposition # diff --git a/components/mpas-albany-landice/bld/build-namelist-section b/components/mpas-albany-landice/bld/build-namelist-section index 4b5e05ec3cd..735dca93c19 100644 --- a/components/mpas-albany-landice/bld/build-namelist-section +++ b/components/mpas-albany-landice/bld/build-namelist-section @@ -182,6 +182,7 @@ add_default($nl, 'config_pio_stride'); add_default($nl, 'config_year_digits'); add_default($nl, 'config_output_external_velocity_solver_data'); add_default($nl, 'config_write_albany_ascii_mesh'); +add_default($nl, 'config_create_all_logs_in_e3sm'); ################################# # Namelist group: decomposition # diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml index 8dc305533ca..3e0b5625835 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml @@ -150,6 +150,7 @@ 4 .false. .false. +.false. 3 diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml index e16e7042a2c..c75c1e163aa 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml @@ -1087,6 +1087,14 @@ Valid values: .true. or .false. Default: Defined in namelist_defaults.xml + +Logical flag determining if log files will be created for each processor in an E3SM configuration. If .true., the model initializes and writes to one files per processor. + +Valid values: .true. or .false. +Default: Defined in namelist_defaults.xml + + diff --git a/components/mpas-albany-landice/driver/glc_comp_mct.F b/components/mpas-albany-landice/driver/glc_comp_mct.F index ac33d17b1f3..6a9df4f406f 100644 --- a/components/mpas-albany-landice/driver/glc_comp_mct.F +++ b/components/mpas-albany-landice/driver/glc_comp_mct.F @@ -181,7 +181,7 @@ subroutine glc_init_mct( EClock, cdata_g, x2g_g, g2x_g, NLFilename )!{{{ logical :: streamsExists integer :: mesh_iotype - logical, pointer :: tempLogicalConfig + logical, pointer :: tempLogicalConfig, config_create_all_logs_in_e3sm character(len=StrKIND), pointer :: tempCharConfig @@ -332,22 +332,6 @@ end subroutine xml_stream_get_attributes call mpas_dmpar_abort(domain % dminfo) end if - ! Set core specific options here - ! Disable output from all but the master task for E3SM! - ! (This overrides the default set by mpas_log_init based on MPAS_DEBUG setting.) - if (iam /= 0) then - domain % logInfo % outputLog % isActive = .false. - endif - - ! After core has had a chance to modify log defaults, open the output log - call mpas_log_open(err=ierr) - if ( ierr /= 0 ) then - write(glcLogUnit,*) 'ERROR: log open failed for core ' // trim(domain % core % coreName) - call mpas_dmpar_abort(domain % dminfo) - end if - ! =========== - - ! ---------- ! Process namelist and streams files ! ---------- @@ -361,6 +345,26 @@ end subroutine xml_stream_get_attributes call mpas_log_write('Namelist setup failed for core ' // trim(domain % core % coreName), MPAS_LOG_CRIT) end if + ! Set core specific options here + ! Disable output from all but the master task for E3SM! + ! (This overrides the default set by mpas_log_init based on MPAS_DEBUG setting.) + call mpas_pool_get_config(domain % configs, 'config_create_all_logs_in_e3sm', config_create_all_logs_in_e3sm) + if (config_create_all_logs_in_e3sm) then + domain % logInfo % outputLog % isActive = .true. + else + if (iam /= 0) then + domain % logInfo % outputLog % isActive = .false. + endif + endif + + ! After core has had a chance to modify log defaults, open the output log + call mpas_log_open(err=ierr) + if ( ierr /= 0 ) then + write(glcLogUnit,*) 'ERROR: log open failed for core ' // trim(domain % core % coreName) + call mpas_dmpar_abort(domain % dminfo) + end if + ! =========== + call mpas_framework_init_phase2(domain, io_system) ! Define package variables diff --git a/components/mpas-albany-landice/src/Registry.xml b/components/mpas-albany-landice/src/Registry.xml index e217bbba2ab..014d8a96591 100644 --- a/components/mpas-albany-landice/src/Registry.xml +++ b/components/mpas-albany-landice/src/Registry.xml @@ -609,6 +609,10 @@ description="Logical flag determining if ascii mesh files will be created. These files are written in a format that can be used by the standalone Albany velocity solver for optimization. If .true., the model initializes, writes the mesh files, and then terminates." possible_values=".true. or .false." /> + From f5664a0d2aa06ca85138c3a78d5e3143c3d4d620 Mon Sep 17 00:00:00 2001 From: noel Date: Thu, 10 Oct 2024 15:32:34 -0700 Subject: [PATCH 157/366] Update pelayout on pm-cpu for ne4 scream tests to ensure 96 tasks. Rename a machinefile to reflect machine name --- cime_config/allactive/config_pesall.xml | 15 +++++++++++++++ .../machine-files/{gcp.cmake => gcp12.cmake} | 0 2 files changed, 15 insertions(+) rename components/eamxx/cmake/machine-files/{gcp.cmake => gcp12.cmake} (100%) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 96ca63f5fbe..0d5f2456596 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -1797,6 +1797,21 @@ + + + allactive+pm-cpu: default, 1 node, 96 tasks, 1 thread + + 96 + 96 + 96 + 96 + 96 + 96 + 96 + 96 + + + diff --git a/components/eamxx/cmake/machine-files/gcp.cmake b/components/eamxx/cmake/machine-files/gcp12.cmake similarity index 100% rename from components/eamxx/cmake/machine-files/gcp.cmake rename to components/eamxx/cmake/machine-files/gcp12.cmake From c34cb650b81d231ccc8bc49e93c317be2b13c97e Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 10 Oct 2024 15:43:33 -0700 Subject: [PATCH 158/366] bug fix for pam_statistics.h --- .../eam/src/physics/crm/pam/pam_statistics.h | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_statistics.h b/components/eam/src/physics/crm/pam/pam_statistics.h index bb277600dfa..8166b9912d2 100644 --- a/components/eam/src/physics/crm/pam/pam_statistics.h +++ b/components/eam/src/physics/crm/pam/pam_statistics.h @@ -458,29 +458,34 @@ inline void pam_statistics_compute_means( pam::PamCoupler &coupler ) { if (clear_rh_cnt(k,iens)>0) { clear_rh(k,iens) = clear_rh(k,iens) / clear_rh_cnt(k,iens); } - phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); - - phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); - - phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); - - phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + if (phys_tend_sgs_cnt(iens)>0) { + phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); + } + if (phys_tend_micro_cnt(iens)>0) { + phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); + } + if (phys_tend_dycor_cnt(iens)>0) { + phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); + } + if (phys_tend_sponge_cnt(iens)>0) { + phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + } }); //------------------------------------------------------------------------------------------------ } From 12c6dfd3f28ade7398e4c443ffa11d7cf2796f1a Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 11 Oct 2024 08:08:33 -0700 Subject: [PATCH 159/366] update PAM submodule --- components/eam/src/physics/crm/pam/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 3ea20ad38f2..c3b6522c57b 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 3ea20ad38f286730973429e0d491420a6f599f11 +Subproject commit c3b6522c57b754d073e1deaad5ce8125f7f88325 From 1b5fb62f8e37402a1a1f2cdf17405ce851112385 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 11 Oct 2024 11:51:37 -0400 Subject: [PATCH 160/366] enable shared builds for gh-standalone --- .github/workflows/eamxx-gh-ci-standalone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index 9fde89ed35b..d276d600357 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -44,7 +44,7 @@ jobs: run: | # TODO: get rid of this extra line if we can? git config --global safe.directory '*' - ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} + ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} -c BUILD_SHARED_LIBS=ON - name: Artifacts uses: actions/upload-artifact@v4 From b8a44d4038ff666e73b075f1d82dc6ce52c14de8 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Fri, 11 Oct 2024 09:30:25 -0700 Subject: [PATCH 161/366] Revert "update fates hydro test module to change sapwood allometry mode" This reverts commit ed346d4bb74a67593308a85e8a7dd8ab5ffec564. --- .../testmods_dirs/elm/fateshydro/shell_commands | 15 --------------- .../testmods_dirs/elm/fateshydro/user_nl_elm | 1 - 2 files changed, 16 deletions(-) diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands index e82b5ce85ff..84648f9222c 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands @@ -1,17 +1,2 @@ #!/bin/bash if [ `./xmlquery --value MACH` == bebop ]; then ./xmlchange --id LND_PIO_TYPENAME --val netcdf; fi -module load e4s -spack env activate gcc -spack load nco - -SRCDIR=`./xmlquery SRCROOT --value` -CASEDIR=`./xmlquery CASEROOT --value` -FATESDIR=$SRCDIR/src/fates -FATESPARAMFILE=$CASEDIR/fates_params_hydrograsstempfix.nc - -ncgen -o $FATESPARAMFILE $FATESDIR/parameter_files/fates_params_default.cdl - -$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_allom_smode --val 1 --allpfts - -spack unload nco -module unload e4s diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm index ea831e5483b..c74b77d05dc 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm @@ -5,7 +5,6 @@ hist_nhtfrq = -24 hist_empty_htapes = .true. use_fates_planthydro = .true. fates_spitfire_mode = 1 -fates_paramfile = '$CASEROOT/fates_params_hydrograsstempfix.nc' hist_fincl1 = 'FATES_ERRH2O_SZPF', 'FATES_TRAN_SZPF', 'FATES_SAPFLOW_SZPF', 'FATES_ITERH1_SZPF','FATES_ABSROOT_H2O_SZPF', 'FATES_TRANSROOT_H2O_SZPF','FATES_STEM_H2O_SZPF','FATES_LEAF_H2O_SZPF', From 9059431e06373527939ac1dde50c4c458f87ff43 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Fri, 11 Oct 2024 12:31:57 -0500 Subject: [PATCH 162/366] Add an fsurdat file for r025 and year 2000 --- components/elm/bld/namelist_files/namelist_defaults.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index cc1bd512e69..bd058707f76 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -360,6 +360,8 @@ lnd/clm2/surfdata_map/surfdata_ne256np4_simyr2010_c220518.nc lnd/clm2/surfdata_map/surfdata_ne256np4_simyr2010_c220518.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr2000_c200624.nc + +lnd/clm2/surfdata_map/surfdata_r025_simyr1980_c240920.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr2000_c190730.nc From 9fe1cd9b7df6b65f3512c766b69ee79c158c99a7 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 11 Oct 2024 15:29:00 -0500 Subject: [PATCH 163/366] Upgrade NetCDF modules for Intel and GNU compilers on Chrysalis Upgraded NetCDF modules on Chrysalis to support CDF5 types in netcdf-fortran. netcdf-c: Updated from 4.4.1 to 4.7.1 (new netcdf-fortran 4.5.3 modules were built with netcdf-c 4.7.1) netcdf-fortran: Updated from 4.4.4 to 4.5.3 (4.4.5 adds nf_64bit_data; 4.5.0 introduces support for unsigned and 64-bit integer types) netcdf-cxx: Removed (not in use, and no modules available built with netcdf-c 4.7.1) Additionally, restored gcc/9.2.0 (required by the specific module intel-mpi/2019.9.304-jdih7h5) for compiler="gnu" and mpilib="impi" configurations. --- cime_config/machines/config_machines.xml | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 140b733f83d..8eb859b9190 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -2644,17 +2644,15 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss openmpi/4.1.6-2mm63n2 hdf5/1.10.7-4cghwvq - netcdf-c/4.4.1-a4hji6e - netcdf-cxx/4.2-ldoxr43 - netcdf-fortran/4.4.4-husened + netcdf-c/4.7.4-4qjdadt + netcdf-fortran/4.5.3-qozrykr parallel-netcdf/1.11.0-icrpxty intel-mpi/2019.9.304-tkzvizk - hdf5/1.8.16-se4xyo7 - netcdf-c/4.4.1-qvxyzq2 - netcdf-cxx/4.2-binixgj - netcdf-fortran/4.4.4-rdxohvp + hdf5/1.10.7-wczt56s + netcdf-c/4.7.4-ba6agmb + netcdf-fortran/4.5.3-5lvy5p4 parallel-netcdf/1.11.0-b74wv4m @@ -2664,17 +2662,19 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss openmpi/4.1.6-ggebj5o hdf5/1.10.7-ol6xuae - netcdf-c/4.4.1-2njo6xx - netcdf-cxx/4.2-7pdzqua - netcdf-fortran/4.4.4-52c6oqi + netcdf-c/4.7.4-pfocec2 + netcdf-fortran/4.5.3-va3hoor parallel-netcdf/1.11.0-d7h4ysd + gcc/11.2.0-bgddrif + intel-oneapi-mkl/2022.1.0-w4kgsn4 + gcc/9.2.0-ugetvbp + intel-mkl/2020.4.304-n3b5fye intel-mpi/2019.9.304-jdih7h5 hdf5/1.8.16-dtbpce3 - netcdf-c/4.4.1-zcoa44z - netcdf-cxx/4.2-ayxg4c7 - netcdf-fortran/4.4.4-2lfr2lr + netcdf-c/4.7.4-seagl7g + netcdf-fortran/4.5.3-ova6t37 parallel-netcdf/1.11.0-ifdodru From 7182613fb62cf7a4014d2e9938a54350f0d0fae8 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 15 Oct 2024 12:18:59 -0400 Subject: [PATCH 164/366] upgrades and fixes to workflows --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 8 ++++++-- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 8 ++++++-- .github/workflows/e3sm-gh-md-linter.yml | 5 +++++ .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml | 5 +++++ .github/workflows/eamxx-gh-ci-standalone.yml | 1 + .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 6 +++++- 8 files changed, 30 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 6c9ee1ab114..5c6ff081f73 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -22,10 +22,14 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: ci: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -36,7 +40,7 @@ jobs: - SMS_D_Ln5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu - ERS_Ld5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.0 steps: - diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index 48c367c8f62..d5687a32b37 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -11,10 +11,14 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: ci-w: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -23,7 +27,7 @@ jobs: - SMS_D_Ld1_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - ERS_Ld3_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.0 steps: - diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 424a871637b..46319b08658 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -10,8 +10,13 @@ on: # for now let's not lint files in eamxx - '!components/eamxx/**/*.md' +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: linter: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ebd2ac9c1e9..dec9bc696bf 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -15,7 +15,7 @@ concurrency: jobs: Build-and-Deploy-docs: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml index 8fe212886d9..fcbf439f409 100644 --- a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml +++ b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml @@ -11,8 +11,13 @@ on: - cron: '00 15 * * 2' workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: mkatmsrffile-test: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index d276d600357..17f79c09eeb 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -18,6 +18,7 @@ on: jobs: ci: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index 2ce70d97843..488c2b11a02 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -34,7 +34,7 @@ concurrency: jobs: eamxx-docs: - if: ${{ github.event.repository.name == 'scream' }} + if: ${{ github.repository == 'E3SM-Project/scream' }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 852bceec535..38c528306c4 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -11,9 +11,13 @@ on: - cron: '00 00 * * *' workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: scream-defaults: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest outputs: event_name: ${{ github.event_name }} From c79b9e4a46910b4e1b7719a7253607ed59787016 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 15 Oct 2024 12:25:32 -0400 Subject: [PATCH 165/366] updates to miniforge settings --- .github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml index fcbf439f409..cacb951b8a8 100644 --- a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml +++ b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml @@ -36,10 +36,7 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: activate-environment: "envmkatmsrffile" - miniforge-variant: Mambaforge miniforge-version: latest - use-mamba: true - mamba-version: "*" channel-priority: strict auto-update-conda: true python-version: 3.11 @@ -47,7 +44,7 @@ jobs: name: Install dependencies run: | echo $CONDA_PREFIX - mamba install -y nco xarray numba numpy netcdf4 + conda install -y nco xarray numba numpy netcdf4 -c conda-forge - name: Run tests working-directory: components/eam/tools/mkatmsrffile From 2dc3601547da390460ef0b408e79a45989dbc8bc Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 15 Oct 2024 12:26:53 -0400 Subject: [PATCH 166/366] add concurrency check for standalone --- .github/workflows/eamxx-gh-ci-standalone.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index 17f79c09eeb..19a2ec9cd8e 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -15,6 +15,10 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: ci: From aef19a4edb4332465ccf531a044df991222783ad Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 15 Oct 2024 12:48:46 -0400 Subject: [PATCH 167/366] rename gh/ci-w to gh-w/ci --- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index d5687a32b37..f51aa88a34c 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -1,4 +1,4 @@ -name: gh +name: gh-w on: pull_request: @@ -17,7 +17,7 @@ concurrency: jobs: - ci-w: + ci: if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: From 321b886377d9b4482cde7fbdff2262c911ad60d0 Mon Sep 17 00:00:00 2001 From: noel Date: Tue, 15 Oct 2024 15:00:38 -0700 Subject: [PATCH 168/366] For pm-cpu, update compiler versions for GCC, nvidia, AMD compilers. For those compilers, update other module versions to now be same as Intel uses. Various updates to muller-cpu/muller-gpu/alvarez. Changes to make machine entries of the NERSC machines more consistent. --- cime_config/machines/config_machines.xml | 117 +++++++++++++++-------- 1 file changed, 78 insertions(+), 39 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 7af65ddcccb..055909533c6 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -190,6 +190,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -218,8 +219,8 @@ PrgEnv-gnu/8.5.0 - gcc/12.2.0 - cray-libsci/23.02.1.1 + gcc-native/12.3 + cray-libsci/23.12.5 @@ -229,35 +230,23 @@ PrgEnv-nvidia - nvidia/22.7 - cray-libsci/23.02.1.1 + nvidia/23.9 + cray-libsci/23.12.5 PrgEnv-aocc - aocc/4.0.0 - cray-libsci/23.02.1.1 + aocc/4.1.0 + cray-libsci/23.12.5 - + craype-accel-host craype/2.7.30 cray-mpich/8.1.28 cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - - - - craype-accel-host - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - - - cmake/3.24.3 evp-patch @@ -367,6 +356,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -377,13 +367,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit - cray-libsci climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -435,6 +426,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -590,12 +582,11 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast - $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} - $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} @@ -603,6 +594,8 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} @@ -618,6 +611,13 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 @@ -684,13 +684,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit - cray-libsci climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -742,6 +743,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -752,6 +754,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -762,6 +765,9 @@ 1 + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} @@ -819,6 +825,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -846,36 +853,35 @@ - PrgEnv-gnu - gcc-native - cray-libsci + PrgEnv-gnu/8.5.0 + gcc-native/13.2 + cray-libsci/24.03.0 - PrgEnv-intel - intel + PrgEnv-intel/8.5.0 + intel/2024.1.0 PrgEnv-nvidia nvidia/24.5 - cray-libsci + cray-libsci/24.03.0 PrgEnv-aocc - aocc/4.0.1 - cray-libsci + aocc/4.1.0 + cray-libsci/24.03.0 craype-accel-host - cray-libsci - craype/2.7.30 - cray-mpich/8.1.28 - cray-hdf5-parallel/1.12.2.9 - cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 + craype/2.7.31.11 + cray-mpich/8.1.29 + cray-hdf5-parallel/1.12.2.11 + cray-netcdf-hdf5parallel/4.9.0.11 + cray-parallel-netcdf/1.12.3.11 cmake/3.24.3 @@ -883,6 +889,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -893,16 +900,48 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} + Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$BLAS_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$BLAS_ROOT"; fi} + $SHELL{if [ -z "$LAPACK_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$LAPACK_ROOT"; fi} + NVHPC + + + Intel10_64_dyn + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 + Spock. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. .*spock.* From 3cf430e36de5b7abc533607ccc464b1709f4979a Mon Sep 17 00:00:00 2001 From: noel Date: Tue, 15 Oct 2024 19:00:06 -0700 Subject: [PATCH 169/366] remove special flags for gnu builds as kdreg2 seems to fix the issue --- .../machines/Depends.muller-cpu.gnu.cmake | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/cime_config/machines/Depends.muller-cpu.gnu.cmake b/cime_config/machines/Depends.muller-cpu.gnu.cmake index 5c7331f979a..53f8b536651 100644 --- a/cime_config/machines/Depends.muller-cpu.gnu.cmake +++ b/cime_config/machines/Depends.muller-cpu.gnu.cmake @@ -7,23 +7,3 @@ if (NOT DEBUG) e3sm_deoptimize_file("${ITEM}") endforeach() endif() - -# On pm-cpu (and muller-cpu), with gcc-native/12.3, we see hang with DEBUG runs of certain tests. -# https://github.com/E3SM-Project/E3SM/issues/6516 -# Currently, we have pm-cpu using gcc/12.2.0 which does not have this issue, but using muller-cpu to test 12.3 -# Turning off -O0 for these 2 files (by adding -O) at least avoids hang and will produce FPE in HOMME code -if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3) - if (DEBUG) - - set(ADJUST - eam/src/dynamics/se/inidat.F90 - eam/src/dynamics/se/dyn_comp.F90 - ) - - foreach(ITEM IN LISTS ADJUST) - e3sm_add_flags("${ITEM}" "-O") - #e3sm_add_flags("${ITEM}" "-DNDEBUG -O") - endforeach() - - endif() -endif() From 224a8397431176cefc20b5c8e9308b2efd9ea46b Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Wed, 16 Oct 2024 09:15:11 -0400 Subject: [PATCH 170/366] load Core/24.07 module and remove versions from git, submodule, and zlib modules --- cime_config/machines/config_machines.xml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 7af65ddcccb..49dcfdb2d25 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1029,6 +1029,7 @@ /usr/share/lmod/lmod/libexec/lmod python + Core/24.07 PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/15.0.1 @@ -1041,6 +1042,7 @@ + Core/24.07 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.4.0 @@ -1049,6 +1051,7 @@ + Core/24.07 PrgEnv-cray PrgEnv-gnu/8.3.3 gcc gcc/12.2.0 @@ -1057,12 +1060,12 @@ rocm/5.4.0 - cray-python/3.9.13.1 + cray-python/3.11.5 cray-libsci - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - zlib/1.2.11 + cmake/3.27.9 + subversion + git + zlib cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 From 9bb456c86da130b8078ed209382d2699f4044178 Mon Sep 17 00:00:00 2001 From: Abhishek Bagusetty Date: Wed, 16 Oct 2024 16:59:00 +0000 Subject: [PATCH 171/366] Update to HOMME Auroro machine cmake config --- components/homme/CMakeLists.txt | 2 +- components/homme/cmake/HommeMacros.cmake | 16 ++++++++---- .../homme/cmake/machineFiles/aurora-aot.cmake | 25 ++++++------------- .../homme/cmake/machineFiles/aurora-jit.cmake | 23 ++++++++--------- .../homme/src/share/cxx/ExecSpaceDefs.cpp | 2 +- .../share/cxx/utilities/scream_tridiag.hpp | 2 +- .../src/share/cxx/vector/vector_pragmas.hpp | 2 +- 7 files changed, 33 insertions(+), 39 deletions(-) diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 6fe81180ab5..26bd2596795 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -258,7 +258,7 @@ endif() OPTION(HOMME_USE_MKL "Whether to use Intel's MKL instead of blas/lapack" FALSE) IF(HOMME_USE_MKL) - MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl will be added to each executable/library.") + MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -qmkl will be added to each executable/library.") ELSE() OPTION(HOMME_FIND_BLASLAPACK "Whether to use system blas/lapack" FALSE) MESSAGE(STATUS "HOMME_FIND_BLASLAPACK=${HOMME_FIND_BLASLAPACK}") diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 6d073dbbe83..68812b9d48a 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -112,7 +112,13 @@ macro(createTestExec execName execType macroNP macroNC ADD_DEFINITIONS(-DHAVE_CONFIG_H) ADD_EXECUTABLE(${execName} ${EXEC_SOURCES}) - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + # For SYCL builds it is suggested to use CXX linker with `-fortlib` + # for mixed-language setups + IF(SYCL_BUILD) + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX) + ELSE() + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + ENDIF() IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() @@ -165,8 +171,8 @@ macro(createTestExec execName execType macroNP macroNC PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR}) IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${execName} PUBLIC -mkl) - TARGET_LINK_LIBRARIES(${execName} -mkl) + TARGET_COMPILE_OPTIONS(${execName} PUBLIC -qmkl) + TARGET_LINK_LIBRARIES(${execName} -qmkl) ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${execName} lapack blas) @@ -264,8 +270,8 @@ macro(createExecLib libName execType libSrcs inclDirs macroNP ENDIF () IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${libName} PUBLIC -mkl) - TARGET_LINK_LIBRARIES(${libName} -mkl) + TARGET_COMPILE_OPTIONS(${libName} PUBLIC -qmkl) + TARGET_LINK_LIBRARIES(${libName} -qmkl) ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${libName} lapack blas) diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake index 094ec888278..14e6d008da8 100644 --- a/components/homme/cmake/machineFiles/aurora-aot.cmake +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -1,11 +1,9 @@ #module restore -#module load oneapi/eng-compiler/2022.12.30.005 -#module load intel_compute_runtime/release/agama-devel-627 -#module load spack cmake +#module load spack-pe-base cmake #module list -SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") +SET (AURORA_MACHINE TRUE CACHE BOOL "") SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") @@ -18,12 +16,12 @@ SET(USE_QUEUING FALSE CACHE BOOL "") #temp hack SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") +SET(HOMME_USE_MKL TRUE CACHE BOOL "") SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") -#set(KOKKOS_HOME "/home/onguba/kokkos-build/mar05-aot/install" CACHE STRING "") -#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") +set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "") SET(USE_TRILINOS OFF CACHE BOOL "") @@ -36,19 +34,14 @@ SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") -# -fsycl-link-huge-device-code for theta to get build -#JIT flags -#SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") -#SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") - #AOT flags SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") -SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=intel_gpu_pvc") SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") -SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") set (ENABLE_OPENMP OFF CACHE BOOL "") @@ -57,8 +50,6 @@ set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") -set (USE_NUM_PROCS 4 CACHE STRING "") - -SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") - +set (USE_NUM_PROCS 12 CACHE STRING "") +SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/aurora-jit.cmake b/components/homme/cmake/machineFiles/aurora-jit.cmake index 1941fa9eb3f..8c35930fdf6 100644 --- a/components/homme/cmake/machineFiles/aurora-jit.cmake +++ b/components/homme/cmake/machineFiles/aurora-jit.cmake @@ -1,10 +1,9 @@ #module restore -#module load oneapi/eng-compiler/2022.12.30.005 -#module load intel_compute_runtime/release/agama-devel-627 -#module load spack cmake +#module load spack-pe-base cmake #module list +SET (AURORA_MACHINE TRUE CACHE BOOL "") SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") @@ -17,12 +16,12 @@ SET(USE_QUEUING FALSE CACHE BOOL "") #temp hack SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") +SET(HOMME_USE_MKL TRUE CACHE BOOL "") SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") -#set(KOKKOS_HOME "/home/onguba/kokkos-build/jan03-2024/install" CACHE STRING "") -#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") +set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "") SET(USE_TRILINOS OFF CACHE BOOL "") @@ -35,14 +34,14 @@ SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") -# -fsycl-link-huge-device-code for theta to get build +#AOT flags SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") -SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl") -SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") -SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") set (ENABLE_OPENMP OFF CACHE BOOL "") @@ -51,8 +50,6 @@ set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") -set (USE_NUM_PROCS 4 CACHE STRING "") - -SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") - +set (USE_NUM_PROCS 12 CACHE STRING "") +SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "") diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index 4f3d97135fe..0b7a3ab34f8 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -22,7 +22,7 @@ #endif #ifdef KOKKOS_ENABLE_SYCL -#include +#include #endif namespace Homme { diff --git a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp index 26221db3955..f302432ef79 100644 --- a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp +++ b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp @@ -120,7 +120,7 @@ int get_team_nthr (const TeamMember& team) { return team.team_size(); } -// Impl details for Nvidia and AMD GPUs. +// Impl details for Nvidia, AMD and Intel GPUs. template KOKKOS_FORCEINLINE_FUNCTION int get_thread_id_within_team_gpu (const TeamMember& team) { diff --git a/components/homme/src/share/cxx/vector/vector_pragmas.hpp b/components/homme/src/share/cxx/vector/vector_pragmas.hpp index 3a0ae7f97ee..b23788a8ccf 100644 --- a/components/homme/src/share/cxx/vector/vector_pragmas.hpp +++ b/components/homme/src/share/cxx/vector/vector_pragmas.hpp @@ -7,7 +7,7 @@ #ifndef HOMMEXX_VECTOR_PRAGMAS_HPP #define HOMMEXX_VECTOR_PRAGMAS_HPP -#if defined(__INTEL_COMPILER) +#if defined(__INTEL_COMPILER) || defined(__INTEL_CLANG_COMPILER) || defined(__INTEL_LLVM_COMPILER) #define VECTOR_IVDEP_LOOP _Pragma("ivdep") #define ALWAYS_VECTORIZE_LOOP _Pragma("vector always") From 442e0686220a47a22e47a406714f4b9444fa5d8a Mon Sep 17 00:00:00 2001 From: Eva Sinha Date: Mon, 16 Oct 2023 12:23:36 -0500 Subject: [PATCH 172/366] Fix for bad leaf C:N ratios during transient land simulations --- components/elm/src/biogeochem/ComputeSeedMod.F90 | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/components/elm/src/biogeochem/ComputeSeedMod.F90 b/components/elm/src/biogeochem/ComputeSeedMod.F90 index 61b916d64f3..292aa036603 100644 --- a/components/elm/src/biogeochem/ComputeSeedMod.F90 +++ b/components/elm/src/biogeochem/ComputeSeedMod.F90 @@ -183,16 +183,10 @@ subroutine LeafProportions(pft_type, ignore_current_state, & pstorage = 0._r8 pxfer = 0._r8 - if (tot_leaf == 0._r8 .or. ignore_current_state) then - if (veg_vp%evergreen(pft_type) == 1._r8) then - pleaf = 1._r8 - else - pstorage = 1._r8 - end if + if (veg_vp%evergreen(pft_type) == 1._r8) then + pleaf = 1._r8 else - pleaf = leaf /tot_leaf - pstorage = leaf_storage/tot_leaf - pxfer = leaf_xfer /tot_leaf + pstorage = 1._r8 end if end subroutine LeafProportions From 1eef7c54ccf5f17d7ef58b460eb512e923066ef5 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Wed, 16 Oct 2024 16:32:47 -0500 Subject: [PATCH 173/366] Update conditionals for Southern Ocean ice and river runoff removal --- components/mpas-ocean/bld/build-namelist | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index 460b9ecda42..69fab9d824e 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -725,12 +725,22 @@ add_default($nl, 'config_sgr_salinity_prescribed'); # Namelist group: coupling # ############################ -add_default($nl, 'config_remove_ais_river_runoff'); -if (($OCN_ICEBERG eq 'true') && ($OCN_FORCING eq 'active_atm')) { +# Assume ice runoff corresponds to icebergs in Southern Ocean (true for JRA +# forcing, perhaps modeled firn ice runoff approximates icebergs). +if ($OCN_ICEBERG eq 'true') { add_default($nl, 'config_remove_ais_ice_runoff', 'val'=>".true."); } else { add_default($nl, 'config_remove_ais_ice_runoff', 'val'=>".false."); } +# Assume river runoff corresponds to ISMF in Southern Ocean only for G-cases +# (true for JRA). When atm active, we assume liquid runoff corresonds to precip +# or snow melt so we do not remove it. In either case, the energy for melting +# doesn't come from the ocean. +if (($OCN_ISMF ne 'none') && ($OCN_FORCING ne 'active_atm')) { + add_default($nl, 'config_remove_ais_river_runoff', 'val'=>".true."); +} else { + add_default($nl, 'config_remove_ais_river_runoff', 'val'=>".false."); +} ###################################### # Namelist group: shortwaveRadiation # From 83dc8ca2459dfa01f41686e71b33b171397555ab Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Thu, 17 Oct 2024 12:44:13 -0600 Subject: [PATCH 174/366] Update components/mpas-ocean/bld/build-namelist --- components/mpas-ocean/bld/build-namelist | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index 69fab9d824e..e38ba077228 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -725,8 +725,7 @@ add_default($nl, 'config_sgr_salinity_prescribed'); # Namelist group: coupling # ############################ -# Assume ice runoff corresponds to icebergs in Southern Ocean (true for JRA -# forcing, perhaps modeled firn ice runoff approximates icebergs). +# When we have a data representation of icebergs, remove ice runoff if ($OCN_ICEBERG eq 'true') { add_default($nl, 'config_remove_ais_ice_runoff', 'val'=>".true."); } else { From ca312d9ff828136f7def5888798bfb39423c962c Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 21 Oct 2024 07:33:04 -0700 Subject: [PATCH 175/366] bump nvidia compiler version on pm-cpu --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 055909533c6..40b3e1b5cc3 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -230,7 +230,7 @@ PrgEnv-nvidia - nvidia/23.9 + nvidia/24.5 cray-libsci/23.12.5 From 8d89f6e61c30aa2576d24cc51a5de0acff16dafe Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 21 Oct 2024 07:51:38 -0700 Subject: [PATCH 176/366] remove kdreg2 change as it's already on master --- cime_config/machines/config_machines.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 40b3e1b5cc3..1b7520a6971 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -582,7 +582,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -754,7 +753,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -900,7 +898,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} From bdcc2f551cfae2fca53bd8aa4ec604601ddf1c68 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 21 Oct 2024 08:14:43 -0700 Subject: [PATCH 177/366] revert removing software as I hit merge conflict --- cime_config/machines/config_machines.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 1b7520a6971..593acc1b067 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -582,6 +582,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + software MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -898,6 +899,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + software MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} From b5207f4917d3273483e5e04fbc0fca44e2c1ac9a Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 21 Oct 2024 09:08:59 -0700 Subject: [PATCH 178/366] Remove work-around to use default --- cime_config/machines/config_machines.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 9142a5fb24e..2d18a00b153 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -277,7 +277,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} @@ -592,7 +591,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software kdreg2 MPI_Bcast $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} @@ -897,7 +895,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} From 238536c739591cd064af8058527cf3efb526d20d Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Tue, 22 Oct 2024 18:04:43 -0500 Subject: [PATCH 179/366] Removing drof modes that point to modified runoff forcing files; modified G-case compsets that were using these drof modes --- .../drof/cime_config/config_component.xml | 24 +----- .../cime_config/namelist_definition_drof.xml | 86 ------------------- .../cime_config/config_compsets.xml | 46 +++++----- 3 files changed, 25 insertions(+), 131 deletions(-) diff --git a/components/data_comps/drof/cime_config/config_component.xml b/components/data_comps/drof/cime_config/config_component.xml index aede006caf2..1c0052d9364 100644 --- a/components/data_comps/drof/cime_config/config_component.xml +++ b/components/data_comps/drof/cime_config/config_component.xml @@ -13,24 +13,14 @@ --> - Data runoff model + Data runoff model NULL mode COREv2 normal year forcing: - COREv2 normal year forcing: - COREv2 normal year forcing: - COREv2 normal year forcing: COREv2 interannual year forcing: - COREv2 interannual year forcing: - COREv2 interannual year forcing: - COREv2 interannual year forcing: CPLHIST mode: JRA55 interannual forcing, v1.5, through 2023 JRA55 interannual forcing, v1.5, through 2020 - JRA55 interannual forcing, v1.5, through 2020, no rofi or rofl around AIS JRA55 interannual forcing, v1.4, through 2018 - JRA55 interannual forcing, v1.4, through 2018, no rofi around AIS - JRA55 interannual forcing, v1.4, through 2018, no rofl around AIS - JRA55 interannual forcing, v1.4, through 2018, no rofi or rofl around AIS JRA55 interannual forcing JRA55 Repeat Year Forcing v1.3 1984-1985 JRA55 Repeat Year Forcing v1.3 1990-1991 @@ -48,27 +38,17 @@ char - CPLHIST,DIATREN_ANN_RX1,DIATREN_ANN_AIS00_RX1,DIATREN_ANN_AIS45_RX1,DIATREN_ANN_AIS55_RX1,DIATREN_IAF_RX1,DIATREN_IAF_AIS00_RX1,DIATREN_IAF_AIS45_RX1,DIATREN_IAF_AIS55_RX1,IAF_JRA,IAF_JRA_1p5,IAF_JRA_1p5_AIS0ROF,IAF_JRA_1p4_2018,IAF_JRA_1p4_2018_AIS0ICE,IAF_JRA_1p4_2018_AIS0LIQ,IAF_JRA_1p4_2018_AIS0ROF,RYF8485_JRA,RYF9091_JRA,RYF0304_JRA,NULL + CPLHIST,DIATREN_ANN_RX1,DIATREN_IAF_RX1,IAF_JRA,IAF_JRA_1p5,IAF_JRA_1p4_2018,RYF8485_JRA,RYF9091_JRA,RYF0304_JRA,NULL DIATREN_ANN_RX1 NULL DIATREN_ANN_RX1 - DIATREN_ANN_AIS00_RX1 - DIATREN_ANN_AIS45_RX1 - DIATREN_ANN_AIS55_RX1 DIATREN_IAF_RX1 - DIATREN_IAF_AIS00_RX1 - DIATREN_IAF_AIS45_RX1 - DIATREN_IAF_AIS55_RX1 CPLHIST IAF_JRA IAF_JRA_1p5 IAF_JRA_1p5 - IAF_JRA_1p5_AIS0ROF IAF_JRA_1p4_2018 - IAF_JRA_1p4_2018_AIS0ICE - IAF_JRA_1p4_2018_AIS0LIQ - IAF_JRA_1p4_2018_AIS0ROF RYF8485_JRA RYF9091_JRA RYF0304_JRA diff --git a/components/data_comps/drof/cime_config/namelist_definition_drof.xml b/components/data_comps/drof/cime_config/namelist_definition_drof.xml index c4139552c70..d4d70074596 100644 --- a/components/data_comps/drof/cime_config/namelist_definition_drof.xml +++ b/components/data_comps/drof/cime_config/namelist_definition_drof.xml @@ -52,18 +52,8 @@ NULL rof.cplhist rof.diatren_ann_rx1 - rof.diatren_ann_ais00_rx1 - rof.diatren_ann_ais45_rx1 - rof.diatren_ann_ais55_rx1 rof.diatren_iaf_rx1 - rof.diatren_iaf_ais00_rx1 - rof.diatren_iaf_ais45_rx1 - rof.diatren_iaf_ais55_rx1 - rof.iaf_jra_1p4_2018_ais0ice - rof.iaf_jra_1p4_2018_ais0liq - rof.iaf_jra_1p4_2018_ais0rof rof.iaf_jra_1p4_2018 - rof.iaf_jra_1p5_ais0rof rof.iaf_jra_1p5 rof.iaf_jra rof.ryf8485_jra @@ -80,13 +70,7 @@ $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 @@ -101,13 +85,7 @@ Stream domain file path(s). runoff.daitren.annual.20190226.nc - runoff.daitren.annual-AISx00.20190226.nc - runoff.daitren.annual-AISx45.20190226.nc - runoff.daitren.annual-AISx55.20190226.nc runoff.daitren.iaf.20120419.nc - runoff.daitren.iaf-AISx00.20120419.nc - runoff.daitren.iaf-AISx45.20120419.nc - runoff.daitren.iaf-AISx55.20120419.nc domain.roff.JRA025.170111.nc domain.roff.JRA025.170111.nc null @@ -134,27 +112,6 @@ arear area mask mask - - time time - xc lon - yc lat - arear area - mask mask - - - time time - xc lon - yc lat - arear area - mask mask - - - time time - xc lon - yc lat - arear area - mask mask - time time domrb_lon lon @@ -173,13 +130,7 @@ $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/ocn/jra55/v1.5_noleap $DIN_LOC_ROOT/lnd/dlnd7/JRA55 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 @@ -194,31 +145,13 @@ Stream data file path(s). runoff.daitren.annual.20190226.nc - runoff.daitren.annual.20190226.nc - runoff.daitren.annual.20190226.nc - runoff.daitren.annual.20190226.nc runoff.daitren.iaf.20120419.nc - runoff.daitren.iaf-AISx00.20120419.nc - runoff.daitren.iaf-AISx45.20120419.nc - runoff.daitren.iaf-AISx55.20120419.nc RAF_8485.JRA.v1.3.runoff.180404.nc RAF_9091.JRA.v1.3.runoff.180404.nc RAF_0304.JRA.v1.3.runoff.180404.nc - - JRA.v1.5.runoff.%y.no_rofi_no_rofl.240411.nc - JRA.v1.5.runoff.%y.240411.nc - - JRA.v1.4.runoff.%y.no_rofi.190214.nc - - - JRA.v1.4.runoff.%y.no_rofl.190214.nc - - - JRA.v1.4.runoff.%y.no_rofi_no_rofl.190214.nc - JRA.v1.4.runoff.%y.190214.nc @@ -244,9 +177,6 @@ runoff rofl - - runoff rofl - rofl rofl rofi rofi @@ -282,7 +212,6 @@ 1 1 1 - 1 $DROF_STRM_YR_ALIGN $DROF_STRM_YR_ALIGN $DROF_CPLHIST_YR_ALIGN @@ -296,13 +225,7 @@ First year of stream. 1 - 1 - 1 - 1 1948 - 1948 - 1948 - 1948 $DROF_STRM_YR_START 1984 1990 @@ -318,18 +241,9 @@ Last year of stream. 1 - 1 - 1 - 1 2009 - 2009 - 2009 - 2009 $DROF_STRM_YR_END $DROF_STRM_YR_END - $DROF_STRM_YR_END - $DROF_STRM_YR_END - $DROF_STRM_YR_END $DROF_STRM_YR_END 1984 1990 diff --git a/components/mpas-ocean/cime_config/config_compsets.xml b/components/mpas-ocean/cime_config/config_compsets.xml index 52ad20a5ab7..62c87ddea3f 100644 --- a/components/mpas-ocean/cime_config/config_compsets.xml +++ b/components/mpas-ocean/cime_config/config_compsets.xml @@ -34,17 +34,17 @@ GMPAS-NYF-PISMF - 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-NYF-PISMF-DSGR - 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%NYFAIS45_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%NYF_SGLC_SWAV GMPAS-NYF-DISMF - 2000_DATM%NYF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%NYF_SGLC_SWAV @@ -94,27 +94,27 @@ GMPAS-JRA1p5-DIB-PISMF - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-PISMF-DSGR - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGR_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGR_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-PISMF-DSGR-TMIX - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGRTMIX_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGRTMIX_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-DISMF - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-PISMF-TMIX - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDTMIX_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDTMIX_DROF%JRA-1p5_SGLC_SWAV @@ -124,77 +124,77 @@ GMPAS-JRA1p4-PISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0LIQ_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0LIQ_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DIB - 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%JRA-1p4-2018-AIS0ICE_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DIB-PISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DIB-DISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-IAF-PISMF - 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-DIB-NYF - 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%NYFAIS55_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-DIB-NYF-PISMF - 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%NYFAIS00_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-DIB-NYF-DISMF - 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%NYFAIS00_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-DIB-IAF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%IAFAIS55_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-DIB-IAF-PISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%IAFAIS00_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-DIB-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%IAFAIS00_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-MALI-DIB-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAFAIS00_MALI%SIASTATIC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAF_MALI%SIASTATIC_SWAV GMPAS-MALI-DIB-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAFAIS00_MALI%SIASTATIC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAF_MALI%SIASTATIC_SWAV From aab68c67e2cbb002adc2ea91d28fd26280afed11 Mon Sep 17 00:00:00 2001 From: Abhishek Bagusetty Date: Wed, 23 Oct 2024 14:30:56 +0000 Subject: [PATCH 180/366] [SYCL] revert -qmkl to -mkl to have backward compatibility with traditional MKL installations. --- components/homme/CMakeLists.txt | 2 +- components/homme/cmake/HommeMacros.cmake | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 26bd2596795..6fe81180ab5 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -258,7 +258,7 @@ endif() OPTION(HOMME_USE_MKL "Whether to use Intel's MKL instead of blas/lapack" FALSE) IF(HOMME_USE_MKL) - MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -qmkl will be added to each executable/library.") + MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl will be added to each executable/library.") ELSE() OPTION(HOMME_FIND_BLASLAPACK "Whether to use system blas/lapack" FALSE) MESSAGE(STATUS "HOMME_FIND_BLASLAPACK=${HOMME_FIND_BLASLAPACK}") diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 68812b9d48a..1466ac5a903 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -171,8 +171,8 @@ macro(createTestExec execName execType macroNP macroNC PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR}) IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${execName} PUBLIC -qmkl) - TARGET_LINK_LIBRARIES(${execName} -qmkl) + TARGET_COMPILE_OPTIONS(${execName} PUBLIC -mkl) + TARGET_LINK_LIBRARIES(${execName} -mkl) ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${execName} lapack blas) @@ -270,8 +270,8 @@ macro(createExecLib libName execType libSrcs inclDirs macroNP ENDIF () IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${libName} PUBLIC -qmkl) - TARGET_LINK_LIBRARIES(${libName} -qmkl) + TARGET_COMPILE_OPTIONS(${libName} PUBLIC -mkl) + TARGET_LINK_LIBRARIES(${libName} -mkl) ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${libName} lapack blas) From 4e2dd44a2516cf1091cfe6fd760ece26fce9a34a Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 23 Oct 2024 08:50:19 -0700 Subject: [PATCH 181/366] Fixes for sw --- .../rrtmgp/scream_rrtmgp_interface.hpp | 70 +++++++++++++------ 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 55e6d87e1fe..8c89a8667af 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -558,9 +558,9 @@ static void rrtmgp_sw( const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { // Get problem sizes - int nbnd = k_dist.get_nband(); - int ngpt = k_dist.get_ngpt(); - int ngas = gas_concs.get_num_gases(); + const int nbnd = k_dist.get_nband(); + const int ngpt = k_dist.get_ngpt(); + const int ngas = gas_concs.get_num_gases(); // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; @@ -601,7 +601,7 @@ static void rrtmgp_sw( }); // Get daytime indices - auto dayIndices = view_t("dayIndices", ncol); + auto dayIndices = pool_t::template alloc(ncol); Kokkos::deep_copy(dayIndices, -1); int nday = 0; @@ -616,24 +616,59 @@ static void rrtmgp_sw( if (nday == 0) { // No daytime columns in this chunk, skip the rest of this routine + pool_t::dealloc(dayIndices); return; } + // Allocate temporaries from pool + const int size1 = nday; + const int size2 = nday*nlay; // 4 + const int size3 = nday*(nlay+1); // 5 + const int size4 = ncol*nlay; + const int size5 = nbnd*nday; //2 + const int size6 = nday*ngpt; + const int size7 = nday*(nlay+1)*nbnd; // 3 + const int size8 = ncol*nlay*(k_dist.get_ngas()+1); + + const int total_size = size1 + size2*4 + size3*5 + size4 + size5*2 + size6 + size7*3 + size8; + auto data = pool_t::template alloc(total_size); RealT* dcurr = data.data(); + + auto mu0_day = view_t (dcurr, nday); dcurr += size1; + + auto p_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto vmr_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_limited = view_t (dcurr, nday, nlay); dcurr += size2; + + auto p_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto t_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_up_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_dir_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + + auto vmr = view_t (dcurr, ncol, nlay); dcurr += size4; + + auto sfc_alb_dir_T = view_t (dcurr, nbnd, nday); dcurr += size5; + auto sfc_alb_dif_T = view_t (dcurr, nbnd, nday); dcurr += size5; + + auto toa_flux = view_t (dcurr, nday, ngpt); dcurr += size6; + + auto bnd_flux_up_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_dir_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + + auto col_gas = view_t(dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size8; + // Subset mu0 - auto mu0_day = view_t("mu0_day", nday); Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { mu0_day(iday) = mu0(dayIndices(iday)); }); // subset state variables - auto p_lay_day = view_t("p_lay_day", nday, nlay); - auto t_lay_day = view_t("t_lay_day", nday, nlay); Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); }); - auto p_lev_day = view_t("p_lev_day", nday, nlay+1); - auto t_lev_day = view_t("t_lev_day", nday, nlay+1); Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); @@ -644,8 +679,6 @@ static void rrtmgp_sw( gas_concs_t gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { - auto vmr_day = view_t("vmr_day", nday, nlay); - auto vmr = view_t("vmr" , ncol, nlay); gas_concs.get_vmr(gas_names[igas], vmr); Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); @@ -677,20 +710,12 @@ static void rrtmgp_sw( // RRTMGP assumes surface albedos have a screwy dimension ordering // for some strange reason, so we need to transpose these; also do // daytime subsetting in the same kernel - view_t sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - view_t sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); Kokkos::parallel_for(MDRP::template get<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); }); // Temporaries we need for daytime-only fluxes - auto flux_up_day = view_t("flux_up_day", nday, nlay+1); - auto flux_dn_day = view_t("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = view_t("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); fluxes_t fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; @@ -710,18 +735,14 @@ static void rrtmgp_sw( } // Limit temperatures for gas optics look-up tables - auto t_lay_limited = view_t("t_lay_limited", nday, nlay); limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); // Do gas optics - view_t toa_flux("toa_flux", nday, ngpt); bool top_at_1 = false; Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { val |= p_lay(0, 0) < p_lay(0, nlay-1); }, Kokkos::LOr(top_at_1)); - view_t col_gas("col_gas", ncol, nlay, k_dist.get_ngas()+1); - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); if (extra_clnsky_diag) { k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); @@ -800,6 +821,9 @@ static void rrtmgp_sw( clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); }); } + + pool_t::dealloc(data); + pool_t::dealloc(dayIndices); } /* From a9c40fd0eae13cea23e88e09216cc72fb2a9dce9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 7 Oct 2024 16:02:09 -0600 Subject: [PATCH 182/366] rrtmgp_lw no allocations --- .../rrtmgp/scream_rrtmgp_interface.hpp | 57 ++++++++++++------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 8c89a8667af..3c3d41119b4 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -840,6 +840,24 @@ static void rrtmgp_lw( { // Problem size int nbnd = k_dist.get_nband(); + int constexpr max_gauss_pts = 4; + + const int size1 = ncol; + const int size2 = nbnd*ncol; + const int size3 = max_gauss_pts*max_gauss_pts; + const int size4 = ncol*nlay; + const int size5 = ncol*(nlay+1); + const int size6 = ncol*nlay*(k_dist.get_ngas()+1); + + RealT* data = pool_t::template alloc_raw(size1 + size2 + size3*2 + size4 + size5 + size6), *dcurr = data; + + view_t t_sfc (dcurr, ncol); dcurr += size1; + view_t emis_sfc (dcurr, nbnd,ncol); dcurr += size2; + view_t gauss_Ds (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t gauss_wts (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t t_lay_limited(dcurr, ncol, nlay); dcurr += size4; + view_t t_lev_limited(dcurr, ncol, nlay+1); dcurr += size5; + view_t col_gas (dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size6; // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; @@ -884,8 +902,6 @@ static void rrtmgp_lw( // Boundary conditions source_func_t lw_sources; lw_sources.alloc(ncol, nlay, k_dist); - view_t t_sfc ("t_sfc" ,ncol); - view_t emis_sfc("emis_sfc",nbnd,ncol); bool top_at_1 = false; Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { @@ -903,32 +919,31 @@ static void rrtmgp_lw( // Weights and angle secants for first order (k=1) Gaussian quadrature. // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - hview_t gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; - gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; - gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; - gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; - - hview_t gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; - gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; - gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; - gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; - - view_t gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - view_t gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + RealT gauss_Ds_host_raw[max_gauss_pts][max_gauss_pts] = { + {1.66, 1.18350343, 1.09719858, 1.06056257}, + {0., 2.81649655, 1.69338507, 1.38282560}, + {0., 0., 4.70941630, 2.40148179}, + {0., 0., 0., 7.15513024} + }; + hview_t gauss_Ds_host (&gauss_Ds_host_raw[0][0], max_gauss_pts, max_gauss_pts); + + RealT gauss_wts_host_raw[max_gauss_pts][max_gauss_pts] = { + {0.5, 0.3180413817, 0.2009319137, 0.1355069134}, + {0., 0.1819586183, 0.2292411064, 0.2034645680}, + {0., 0., 0.0698269799, 0.1298475476}, + {0., 0., 0., 0.0311809710} + }; + + hview_t gauss_wts_host(&gauss_wts_host_raw[0][0],max_gauss_pts,max_gauss_pts); + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); Kokkos::deep_copy(gauss_wts, gauss_wts_host); // Limit temperatures for gas optics look-up tables - auto t_lay_limited = view_t("t_lay_limited", ncol, nlay); - auto t_lev_limited = view_t("t_lev_limited", ncol, nlay+1); limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); // Do gas optics - view_t col_gas("col_gas", ncol, nlay, k_dist.get_ngas()+1); k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); if (extra_clnsky_diag) { k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); @@ -962,6 +977,8 @@ static void rrtmgp_lw( // Compute clean-sky fluxes rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); } + + pool_t::dealloc(data, dcurr - data); } /* From 1a701dd3edf2a497081fe94196c79d67f757941c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 23 Oct 2024 10:18:50 -0700 Subject: [PATCH 183/366] Final pool allocs --- .../rrtmgp/scream_rrtmgp_interface.hpp | 68 +++++++++++++------ 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 3c3d41119b4..3eb015d0424 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -849,7 +849,8 @@ static void rrtmgp_lw( const int size5 = ncol*(nlay+1); const int size6 = ncol*nlay*(k_dist.get_ngas()+1); - RealT* data = pool_t::template alloc_raw(size1 + size2 + size3*2 + size4 + size5 + size6), *dcurr = data; + const int total_size = size1 + size2 + size3*2 + size4 + size5 + size6; + auto data = pool_t::template alloc(total_size); RealT *dcurr = data.data(); view_t t_sfc (dcurr, ncol); dcurr += size1; view_t emis_sfc (dcurr, nbnd,ncol); dcurr += size2; @@ -978,24 +979,22 @@ static void rrtmgp_lw( rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); } - pool_t::dealloc(data, dcurr - data); + pool_t::dealloc(data); } /* * Return a subcolumn mask consistent with a specified overlap assumption */ -static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds) +template +static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const CldfT &cldf, const int overlap_option, const SeedsT &seeds, const SubcT& subcolumn_mask) { - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - int3dk subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); - // Subcolumn generators are a means for producing a variable x(i,j,k), where // // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) // // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = view_t("cldx", ncol, nlay, ngpt); + auto cldx = pool_t::template alloc(ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1055,7 +1054,8 @@ static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, subcolumn_mask(icol,ilay,igpt) = 0; } }); - return subcolumn_mask; + + pool_t::dealloc(cldx); } /* @@ -1067,7 +1067,7 @@ static void compute_cloud_area( { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = view_t("subcol_mask", ncol, ngpt); + auto subcol_mask = pool_t::template alloc(ncol, ngpt); Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but // using play/pmid does not @@ -1084,6 +1084,8 @@ static void compute_cloud_area( cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; } }); + + pool_t::dealloc(subcol_mask); } /* @@ -1116,7 +1118,7 @@ static void compute_aerocom_cloudtop( Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = view_t("aerocom_clr", ncol); + auto aerocom_clr = pool_t::template alloc(ncol); Kokkos::deep_copy(aerocom_clr, 1.0); // Get gravity acceleration constant from constants @@ -1189,6 +1191,8 @@ static void compute_aerocom_cloudtop( // (their products) cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); }); + + pool_t::dealloc(aerocom_clr); } /* @@ -1291,14 +1295,17 @@ static optical_props2_t get_cloud_optics_sw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = view_t("rel_limited", ncol, nlay); - auto rei_limited = view_t("rei_limited", ncol, nlay); + auto rel_limited = pool_t::template alloc(ncol, nlay); + auto rei_limited = pool_t::template alloc(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + // Return optics return clouds; } @@ -1317,14 +1324,17 @@ static optical_props1_t get_cloud_optics_lw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = view_t("rel_limited", ncol, nlay); - auto rei_limited = view_t("rei_limited", ncol, nlay); + auto rel_limited = pool_t::template alloc(ncol, nlay); + auto rei_limited = pool_t::template alloc(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + // Return optics return clouds; } @@ -1336,6 +1346,10 @@ static optical_props2_t get_subsampled_clouds( optical_props2_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_2str(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::template alloc(ncol, nlay, ngpt); + // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud @@ -1343,7 +1357,7 @@ static optical_props2_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + auto cldfrac_rad = pool_t::template alloc(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1357,11 +1371,11 @@ static optical_props2_t get_subsampled_clouds( int overlap = 1; // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = view_t("seeds", ncol); + auto seeds = pool_t::template alloc(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { @@ -1376,6 +1390,11 @@ static optical_props2_t get_subsampled_clouds( subsampled_optics.g (icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + return subsampled_optics; } @@ -1387,6 +1406,10 @@ static optical_props1_t get_subsampled_clouds( optical_props1_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_1scl(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::template alloc(ncol, nlay, ngpt); + // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud @@ -1394,7 +1417,7 @@ static optical_props1_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + auto cldfrac_rad = pool_t::template alloc(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1405,11 +1428,11 @@ static optical_props1_t get_subsampled_clouds( // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM; use different // seed values for longwave and shortwave - auto seeds = view_t("seeds", ncol); + auto seeds = pool_t::template alloc(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { @@ -1420,6 +1443,11 @@ static optical_props1_t get_subsampled_clouds( subsampled_optics.tau(icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + return subsampled_optics; } From 784223887493e92c4f9913a7f4d76f1b85e195da Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 23 Oct 2024 10:41:03 -0700 Subject: [PATCH 184/366] Fix unit test --- .../eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 99a58244561..00a33c0380d 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -1291,7 +1291,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds, cldmask); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { @@ -1325,7 +1325,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds, cldmask); auto cldmask_h = chc(cldmask); for (int igpt = 0; igpt < ngpt; igpt++) { if (cldmask_h(0,0,igpt) == 1) { From afbefb4625b72ef102505cf812c8bf4c110547ad Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 23 Oct 2024 12:14:14 -0700 Subject: [PATCH 185/366] Something went wrong with merge --- .../rrtmgp/scream_rrtmgp_interface.hpp | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index ecdaf7cb507..3eb015d0424 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -562,44 +562,6 @@ static void rrtmgp_sw( const int ngpt = k_dist.get_ngpt(); const int ngas = gas_concs.get_num_gases(); - // Allocate temporaries from pool - const int size1 = nday; - const int size2 = nday*nlay; // 4 - const int size3 = nday*(nlay+1); // 5 - const int size4 = ncol*nlay; - const int size5 = nbnd*nday; //2 - const int size6 = nday*ngpt; - const int size7 = nday*(nlay+1)*nbnd; // 3 - const int size8 = ncol*nlay*(k_dist.get_ngas()+1); - - RealT* data = pool_t::template alloc_raw(size1 + size2*4 + size3*5 + size4 + size5*2 + size6 + size7*3 + size8), *dcurr = data; - - auto mu0_day = view_t (dcurr, nday); dcurr += size1; - - auto p_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; - auto t_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; - auto vmr_day = view_t (dcurr, nday, nlay); dcurr += size2; - auto t_lay_limited = view_t (dcurr, nday, nlay); dcurr += size2; - - auto p_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - auto t_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - auto flux_up_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - auto flux_dn_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - auto flux_dn_dir_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - - auto vmr = view_t (dcurr, ncol, nlay); dcurr += size4; - - auto sfc_alb_dir_T = view_t (dcurr, nbnd, nday); dcurr += size5; - auto sfc_alb_dif_T = view_t (dcurr, nbnd, nday); dcurr += size5; - - auto toa_flux = view_t (dcurr, nday, ngpt); dcurr += size6; - - auto bnd_flux_up_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; - auto bnd_flux_dn_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; - auto bnd_flux_dn_dir_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; - - auto col_gas = view_t(dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size8; - // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; auto &flux_dn = fluxes.flux_dn; From ce98c5a075a3ebf4293fb923ce8ea3bc5962047b Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 11 Oct 2024 17:46:28 -0600 Subject: [PATCH 186/366] Update chicoma-cpu modules --- cime_config/machines/config_machines.xml | 50 ++++++++++++++---------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index acd10f74110..b882536897e 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4003,11 +4003,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4019,26 +4019,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 + PrgEnv-gnu/8.5.0 gcc/12.2.0 cray-libsci/23.05.1.4 @@ -4050,8 +4051,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 @@ -4060,15 +4061,22 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-libsci/23.05.1.4 - + + craype-accel-host + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 + + + craype-accel-host craype/2.7.21 cray-mpich/8.1.26 - libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 @@ -4090,6 +4098,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + -1 @@ -4131,11 +4142,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4202,7 +4213,6 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 From 457c9439ad86c41a8ef80db04b086e3d8957983c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 14 Oct 2024 14:43:58 -0600 Subject: [PATCH 187/366] Add latest cmake --- cime_config/machines/config_machines.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index b882536897e..3c20372aa53 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4068,6 +4068,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 + cmake/3.27.7 @@ -4077,6 +4078,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + cmake/3.27.7 @@ -4213,6 +4215,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + cmake/3.27.7 From a44f71e8c87618bd43961529feb6378306ca5a26 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 17 Oct 2024 09:34:47 -0600 Subject: [PATCH 188/366] Updates to get E3SM working on chicoma-cpu --- cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake | 1 + cime_config/machines/config_machines.xml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake index a6148451eb7..a6c13942620 100644 --- a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake @@ -5,6 +5,7 @@ endif() set(PIO_FILESYSTEM_HINTS "lustre") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--enable-new-dtags") set(MPICC "cc") set(MPICXX "CC") set(MPIFC "ftn") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 3c20372aa53..f893663f0ad 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3983,7 +3983,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -4068,7 +4068,6 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - cmake/3.27.7 @@ -4078,6 +4077,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + + + cmake/3.27.7 From 699dea0bf09e638ee9a897381e3528cee44821a0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 19 Oct 2024 15:07:53 -0600 Subject: [PATCH 189/366] Update nvidia modules --- cime_config/machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index f893663f0ad..941bcef5170 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4045,8 +4045,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - PrgEnv-nvidia/8.4.0 - nvidia/22.7 + PrgEnv-nvidia/8.5.0 + nvidia/24.7 cray-libsci/23.05.1.4 From 6453565d37e8bc24930c753fa5b9ff4867b42f85 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 18:31:01 -0600 Subject: [PATCH 190/366] Add GNU_CRAY_LDFLAGS flag to MPAS standalone builds This should be set to: ``` export GNU_CRAY_LDFLAGS="-Wl,--enable-new-dtags" ``` on Chicoma-CPU with gcc. --- components/mpas-framework/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 1f9b00844e7..f92606987bb 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -396,11 +396,11 @@ gnu-cray: "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ - "LDFLAGS_OPT = -O3 -m64" \ + "LDFLAGS_OPT = -O3 -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -g -m64" \ - "LDFLAGS_DEBUG = -g -m64" \ + "LDFLAGS_DEBUG = -g -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_OMP = -fopenmp" \ "CFLAGS_OMP = -fopenmp" \ "BUILD_TARGET = $(@)" \ From 750707a82541eb245db50da51f0a686afcd14536 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 21:43:12 -0600 Subject: [PATCH 191/366] Revert chicoma-gpu changes (for now) --- cime_config/machines/config_machines.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 941bcef5170..3964b1405a2 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4146,11 +4146,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/lmod/init/perl + /usr/share/lmod/8.3.1/init/perl - /usr/share/lmod/lmod/init/python - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh + /usr/share/lmod/8.3.1/init/python + /usr/share/lmod/8.3.1/init/sh + /usr/share/lmod/8.3.1/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4217,7 +4217,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.27.7 + cmake/3.25.1 From 0cb78eaff39e5290982f59e9eaf840f26de6a3c3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 21 Oct 2024 11:44:41 -0600 Subject: [PATCH 192/366] Update gcc and nvidia to newer versions This is to match proposed updates to Perlmutter-CPU https://github.com/E3SM-Project/E3SM/pull/6702 --- cime_config/machines/config_machines.xml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 3964b1405a2..b1df10a96e1 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4019,6 +4019,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 + gcc + gcc-native intel intel-oneapi nvidia @@ -4040,14 +4042,14 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss PrgEnv-gnu/8.5.0 - gcc/12.2.0 - cray-libsci/23.05.1.4 + gcc-native/12.3 + cray-libsci/23.12.5 PrgEnv-nvidia/8.5.0 nvidia/24.7 - cray-libsci/23.05.1.4 + cray-libsci/23.12.5 @@ -4061,25 +4063,13 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-libsci/23.05.1.4 - + craype-accel-host craype/2.7.30 cray-mpich/8.1.28 cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - - - - craype-accel-host - craype/2.7.21 - cray-mpich/8.1.26 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - - - cmake/3.27.7 @@ -4103,7 +4093,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_PARALLEL_NETCDF_PREFIX} - /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + /usr/lib64/gcc/x86_64-suse-linux/12:$ENV{LD_LIBRARY_PATH} -1 From 461066e623d36a4cee51890ec89eba9b4d92fa5a Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 24 Oct 2024 11:51:20 -0600 Subject: [PATCH 193/366] Fix some pool issues --- components/eam/src/physics/rrtmgp/external | 2 +- .../physics/rrtmgp/scream_rrtmgp_interface.hpp | 2 +- .../physics/rrtmgp/tests/rrtmgp_unit_tests.cpp | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index b851a86aa93..b24ca1f616e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit b851a86aa9312193f8f292f612e40afb056f90cf +Subproject commit b24ca1f616e45659b334dbd7297017cb7927367e diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 3eb015d0424..0d890317b99 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -1067,7 +1067,7 @@ static void compute_cloud_area( { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = pool_t::template alloc(ncol, ngpt); + auto subcol_mask = pool_t::template alloc_and_init(ncol, ngpt); Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but // using play/pmid does not diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 00a33c0380d..06ca64af5f2 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -850,6 +850,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop") { #ifdef RRTMGP_ENABLE_KOKKOS using interface_t = scream::rrtmgp::rrtmgp_interface<>; +using pool_t = interface_t::pool_t; using real1dk = interface_t::view_t; using real2dk = interface_t::view_t; using real3dk = interface_t::view_t; @@ -861,6 +862,7 @@ using MDRP = interface_t::MDRP; TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos scream::init_kls(); + pool_t::init(10000); // Test heating rate function by passing simple inputs auto dp = real2dk("dp", 1, 1); @@ -908,12 +910,14 @@ TEST_CASE("rrtmgp_test_heating_k") { REQUIRE(chc(heating)(0,0) == heating_ref); // Clean up + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); using physconst = scream::physics::Constants; @@ -966,12 +970,14 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Clean up + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_test_limit_to_bounds_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Test limiter function auto arr = real2dk("arr", 2, 2); @@ -998,6 +1004,8 @@ TEST_CASE("rrtmgp_test_limit_to_bounds_k") { REQUIRE(chc(arr_limited)(0,1) == 2.0); REQUIRE(chc(arr_limited)(1,0) == 3.0); REQUIRE(chc(arr_limited)(1,1) == 3.5); + + pool_t::finalize(); scream::finalize_kls(); } @@ -1070,6 +1078,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create arrays const int ncol = 1; @@ -1227,6 +1236,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { logger->info("Free memory...\n"); interface_t::rrtmgp_finalize(); gas_concs.reset(); + pool_t::finalize(); scream::finalize_kls(); } @@ -1255,6 +1265,7 @@ TEST_CASE("rrtmgp_test_radiation_do_k") { TEST_CASE("rrtmgp_test_check_range_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create some dummy data and test with both values inside valid range and outside auto dummy = real2dk("dummy", 2, 1); // All values within range @@ -1266,12 +1277,14 @@ TEST_CASE("rrtmgp_test_check_range_k") { // At least one value above upper bound Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 0) = 1.1;}); REQUIRE(scream::rrtmgp::check_range_k(dummy, 0.0, 1.0, "dummy") == false); + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_test_subcol_gen_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create dummy data const int ncol = 1; const int nlay = 4; @@ -1334,12 +1347,14 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { } } // Clean up after test + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_cloud_area_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create dummy data const int ncol = 1; const int nlay = 2; @@ -1429,12 +1444,14 @@ TEST_CASE("rrtmgp_cloud_area_k") { REQUIRE(chc(cldtot)(0) == 0.0); interface_t::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_aerocom_cloudtop_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create dummy data const int ncol = 1; @@ -1623,6 +1640,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { REQUIRE(chc(cldfrac_ice_at_cldtop)(0) == 0.7); // max // cleanup + pool_t::finalize(); scream::finalize_kls(); } #endif From 8ee29b50e65013d4cb892020ed130b160ff497da Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 24 Oct 2024 11:58:12 -0600 Subject: [PATCH 194/366] alloc and init everything --- .../rrtmgp/scream_rrtmgp_interface.hpp | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 0d890317b99..03cb0693d98 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -601,7 +601,7 @@ static void rrtmgp_sw( }); // Get daytime indices - auto dayIndices = pool_t::template alloc(ncol); + auto dayIndices = pool_t::template alloc_and_init(ncol); Kokkos::deep_copy(dayIndices, -1); int nday = 0; @@ -631,7 +631,7 @@ static void rrtmgp_sw( const int size8 = ncol*nlay*(k_dist.get_ngas()+1); const int total_size = size1 + size2*4 + size3*5 + size4 + size5*2 + size6 + size7*3 + size8; - auto data = pool_t::template alloc(total_size); RealT* dcurr = data.data(); + auto data = pool_t::template alloc_and_init(total_size); RealT* dcurr = data.data(); auto mu0_day = view_t (dcurr, nday); dcurr += size1; @@ -850,7 +850,7 @@ static void rrtmgp_lw( const int size6 = ncol*nlay*(k_dist.get_ngas()+1); const int total_size = size1 + size2 + size3*2 + size4 + size5 + size6; - auto data = pool_t::template alloc(total_size); RealT *dcurr = data.data(); + auto data = pool_t::template alloc_and_init(total_size); RealT *dcurr = data.data(); view_t t_sfc (dcurr, ncol); dcurr += size1; view_t emis_sfc (dcurr, nbnd,ncol); dcurr += size2; @@ -994,7 +994,7 @@ static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, c // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) // // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = pool_t::template alloc(ncol, nlay, ngpt); + auto cldx = pool_t::template alloc_and_init(ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1118,7 +1118,7 @@ static void compute_aerocom_cloudtop( Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = pool_t::template alloc(ncol); + auto aerocom_clr = pool_t::template alloc_and_init(ncol); Kokkos::deep_copy(aerocom_clr, 1.0); // Get gravity acceleration constant from constants @@ -1295,8 +1295,8 @@ static optical_props2_t get_cloud_optics_sw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = pool_t::template alloc(ncol, nlay); - auto rei_limited = pool_t::template alloc(ncol, nlay); + auto rel_limited = pool_t::template alloc_and_init(ncol, nlay); + auto rei_limited = pool_t::template alloc_and_init(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); @@ -1324,8 +1324,8 @@ static optical_props1_t get_cloud_optics_lw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = pool_t::template alloc(ncol, nlay); - auto rei_limited = pool_t::template alloc(ncol, nlay); + auto rel_limited = pool_t::template alloc_and_init(ncol, nlay); + auto rei_limited = pool_t::template alloc_and_init(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); @@ -1348,7 +1348,7 @@ static optical_props2_t get_subsampled_clouds( subsampled_optics.alloc_2str(ncol, nlay); // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto cldmask = pool_t::template alloc(ncol, nlay, ngpt); + auto cldmask = pool_t::template alloc_and_init(ncol, nlay, ngpt); // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" @@ -1357,7 +1357,7 @@ static optical_props2_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = pool_t::template alloc(ncol, nlay); + auto cldfrac_rad = pool_t::template alloc_and_init(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1371,7 +1371,7 @@ static optical_props2_t get_subsampled_clouds( int overlap = 1; // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = pool_t::template alloc(ncol); + auto seeds = pool_t::template alloc_and_init(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); }); @@ -1408,7 +1408,7 @@ static optical_props1_t get_subsampled_clouds( subsampled_optics.alloc_1scl(ncol, nlay); // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto cldmask = pool_t::template alloc(ncol, nlay, ngpt); + auto cldmask = pool_t::template alloc_and_init(ncol, nlay, ngpt); // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" @@ -1417,7 +1417,7 @@ static optical_props1_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = pool_t::template alloc(ncol, nlay); + auto cldfrac_rad = pool_t::template alloc_and_init(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1428,7 +1428,7 @@ static optical_props1_t get_subsampled_clouds( // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM; use different // seed values for longwave and shortwave - auto seeds = pool_t::template alloc(ncol); + auto seeds = pool_t::template alloc_and_init(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); }); From 85cd5d3fa02b2e042cd571ed61dc20536612738d Mon Sep 17 00:00:00 2001 From: noel Date: Thu, 24 Oct 2024 14:49:00 -0700 Subject: [PATCH 195/366] Add overrun qos for pm-cpu/pm-gpu --- cime_config/machines/config_batch.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 923e497367f..ee271d1f9dc 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -448,6 +448,7 @@ regular preempt shared + overrun debug @@ -502,10 +503,11 @@ regular preempt shared + overrun debug - + --constraint=cpu From 360692f3004c740a542c00867f7c726e1ef56629 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 25 Oct 2024 13:51:46 -0600 Subject: [PATCH 196/366] Way more pool mem was being allocated than was needed --- components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 03cb0693d98..b96477c5a3f 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -240,7 +240,7 @@ static void rrtmgp_initialize( load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); // initialize kokkos rrtmgp pool allocator - const size_t base_ref = 18000; + const size_t base_ref = 4000; const size_t ncol = gas_concs.ncol; const size_t nlay = gas_concs.nlay; const size_t nlev = SCREAM_NUM_VERTICAL_LEV; From 1634f028d1781dc9e4a9bf45f741473fdaa06409 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 25 Oct 2024 14:22:22 -0600 Subject: [PATCH 197/366] Moved base_ref in the wrong direction --- components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index b96477c5a3f..e807643b05a 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -240,7 +240,7 @@ static void rrtmgp_initialize( load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); // initialize kokkos rrtmgp pool allocator - const size_t base_ref = 4000; + const size_t base_ref = 80000; const size_t ncol = gas_concs.ncol; const size_t nlay = gas_concs.nlay; const size_t nlev = SCREAM_NUM_VERTICAL_LEV; From 3a2d60d02bf09165546ebba2efe228ff65c15137 Mon Sep 17 00:00:00 2001 From: Abhishek Bagusetty Date: Wed, 30 Oct 2024 16:35:20 +0000 Subject: [PATCH 198/366] [Aurora] Fix MKL flags from Intel MKL or oneMKL --- components/homme/CMakeLists.txt | 14 ++++++++++++-- components/homme/cmake/HommeMacros.cmake | 18 ++++++++++++------ .../homme/test/unit_tests/CMakeLists.txt | 11 +++++++---- .../test_execs/preqx_kokkos_ut/CMakeLists.txt | 8 ++++++-- .../test_execs/thetal_kokkos_ut/CMakeLists.txt | 8 ++++++-- 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 6fe81180ab5..121b9780e7e 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -256,9 +256,19 @@ elseif (NOT TARGET csm_share) message (FATAL_ERROR "Aborting.") endif() -OPTION(HOMME_USE_MKL "Whether to use Intel's MKL instead of blas/lapack" FALSE) +OPTION(HOMME_USE_MKL "Whether to use Intel's MKL/oneMKL instead of blas/lapack" FALSE) IF(HOMME_USE_MKL) - MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl will be added to each executable/library.") + find_package(MKL REQUIRED $ENV{MKLROOT}/lib/cmake/mkl) + + # Check if the path contains "oneapi/mkl" which indicates oneMKL + if("$ENV{MKLROOT}" MATCHES "oneapi/mkl") + set(MKL_TYPE "oneMKL") + message(STATUS "Using oneMKL from Intel oneAPI.") + else() + set(MKL_TYPE "Intel MKL") + message(STATUS "Using standalone Intel MKL.") + endif() + MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl/-qmkl will be added to each executable/library.") ELSE() OPTION(HOMME_FIND_BLASLAPACK "Whether to use system blas/lapack" FALSE) MESSAGE(STATUS "HOMME_FIND_BLASLAPACK=${HOMME_FIND_BLASLAPACK}") diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 1466ac5a903..29a3c64bba9 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -162,17 +162,20 @@ macro(createTestExec execName execType macroNP macroNC ENDIF () IF (HOMME_USE_KOKKOS) - target_link_libraries(${execName} Kokkos::kokkos) + TARGET_LINK_LIBRARIES(${execName} Kokkos::kokkos) ENDIF () # Move the module files out of the way so the parallel build # doesn't have a race condition SET_TARGET_PROPERTIES(${execName} - PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR}) + PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR}) IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${execName} PUBLIC -mkl) - TARGET_LINK_LIBRARIES(${execName} -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(${execName} -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(${execName} -mkl) + ENDIF () ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${execName} lapack blas) @@ -270,8 +273,11 @@ macro(createExecLib libName execType libSrcs inclDirs macroNP ENDIF () IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${libName} PUBLIC -mkl) - TARGET_LINK_LIBRARIES(${libName} -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(${libName} -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(${libName} -mkl) + ENDIF () ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${libName} lapack blas) diff --git a/components/homme/test/unit_tests/CMakeLists.txt b/components/homme/test/unit_tests/CMakeLists.txt index 2b601ae172c..b029e73277e 100644 --- a/components/homme/test/unit_tests/CMakeLists.txt +++ b/components/homme/test/unit_tests/CMakeLists.txt @@ -22,14 +22,17 @@ macro(cxx_unit_test target_name target_f90_srcs target_cxx_srcs include_dirs con cxx_unit_test_add_test(${target_name}_test ${target_name} ${NUM_CPUS}) TARGET_LINK_LIBRARIES(${target_name} timing ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) - target_link_libraries(${target_name} Kokkos::kokkos) + TARGET_LINK_LIBRARIES(${target_name} Kokkos::kokkos) IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS (${target_name} PUBLIC -mkl) - TARGET_LINK_LIBRARIES (${target_name} -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(${target_name} -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(${target_name} -mkl) + ENDIF () ENDIF() # Link csm_share lib - target_link_libraries(${target_name} csm_share) + TARGET_LINK_LIBRARIES(${target_name} csm_share) STRING(TOUPPER "${PERFORMANCE_PROFILE}" PERF_PROF_UPPER) IF ("${PERF_PROF_UPPER}" STREQUAL "VTUNE") diff --git a/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt index 9dd30f58f63..3347e7b4894 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt @@ -36,10 +36,14 @@ ADD_LIBRARY(preqx_kokkos_ut_lib TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS}) TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) TARGET_COMPILE_DEFINITIONS(preqx_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H") -target_link_libraries(preqx_kokkos_ut_lib Kokkos::kokkos) +TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib Kokkos::kokkos) TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib timing csm_share ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) IF (HOMME_USE_MKL) - TARGET_LINK_LIBRARIES (preqx_kokkos_ut_lib -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib -mkl) + ENDIF () ENDIF() IF(NOT BUILD_HOMME_WITHOUT_PIOLIBRARY) IF(HOMME_USE_SCORPIO) diff --git a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt index e8bf5e20bd0..11030b15e87 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt @@ -42,10 +42,14 @@ ADD_LIBRARY(thetal_kokkos_ut_lib TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS}) TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) TARGET_COMPILE_DEFINITIONS(thetal_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H") -target_link_libraries(thetal_kokkos_ut_lib Kokkos::kokkos) +TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib Kokkos::kokkos) TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib timing csm_share ${COMPOSE_LIBRARY_CPP} ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) IF (HOMME_USE_MKL) - TARGET_LINK_LIBRARIES (thetal_kokkos_ut_lib -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib -mkl) + ENDIF () ENDIF() IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(thetal_kokkos_ut_lib PUBLIC HOMME_WITHOUT_PIOLIBRARY) From e9833e4d818a804c5584d42e9d2137d5d94ad652 Mon Sep 17 00:00:00 2001 From: Abhishek Bagusetty Date: Wed, 30 Oct 2024 19:55:09 +0000 Subject: [PATCH 199/366] [Aurora] set MKL package for only onemkl and not intel mkl --- components/homme/CMakeLists.txt | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 121b9780e7e..ec291d8bd26 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -258,16 +258,19 @@ endif() OPTION(HOMME_USE_MKL "Whether to use Intel's MKL/oneMKL instead of blas/lapack" FALSE) IF(HOMME_USE_MKL) - find_package(MKL REQUIRED $ENV{MKLROOT}/lib/cmake/mkl) - - # Check if the path contains "oneapi/mkl" which indicates oneMKL - if("$ENV{MKLROOT}" MATCHES "oneapi/mkl") - set(MKL_TYPE "oneMKL") - message(STATUS "Using oneMKL from Intel oneAPI.") - else() - set(MKL_TYPE "Intel MKL") - message(STATUS "Using standalone Intel MKL.") - endif() + IF(DEFINED ENV{MKLROOT}) + SET(MKL_ROOT_PATH "$ENV{MKLROOT}") + IF(MKL_ROOT_PATH MATCHES "oneapi/mkl") + SET(MKL_TYPE "oneMKL") + MESSAGE(STATUS "Detected oneMKL based on MKLROOT: ${MKL_ROOT_PATH}") + FIND_PACKAGE(MKL REQUIRED $ENV{MKLROOT}/lib/cmake/mkl) + ELSE() + SET(MKL_TYPE "Intel MKL") + MESSAGE(STATUS "Detected standalone Intel MKL based on MKLROOT: ${MKL_ROOT_PATH}") + ENDIF() + ELSE() + MESSAGE(FATAL_ERROR "MKLROOT environment variable is not set. Please set it to your MKL installation path.") + ENDIF() MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl/-qmkl will be added to each executable/library.") ELSE() OPTION(HOMME_FIND_BLASLAPACK "Whether to use system blas/lapack" FALSE) From a54cc59e51f4e21b3c8f04b4a0c99241b522ec3c Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Wed, 30 Oct 2024 20:07:54 -0400 Subject: [PATCH 200/366] EAMxx: Add an REP L128 test. --- cime_config/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 3c89e8e9099..f76ef7a0ce2 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -706,7 +706,8 @@ "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-spa_remap--scream-output-preset-4", "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash--scream-output-preset-6", - "ERS_Ln22.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-L128--scream-output-preset-4" + "ERS_Ln22.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-L128--scream-output-preset-4", + "REP_Ld5.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-L128--scream-output-preset-6" ) }, From 4bc46ebbfe35f0cd39b687afba1e5f82990035e6 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 31 Oct 2024 09:09:03 -0700 Subject: [PATCH 201/366] Reset alarm for MALI adaptive timestepper force interval in GLC driver This is necessary for the MALI adaptive timestepper to work without error in E3SM. --- components/mpas-albany-landice/driver/glc_comp_mct.F | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/mpas-albany-landice/driver/glc_comp_mct.F b/components/mpas-albany-landice/driver/glc_comp_mct.F index 6a9df4f406f..a3eb5d17924 100644 --- a/components/mpas-albany-landice/driver/glc_comp_mct.F +++ b/components/mpas-albany-landice/driver/glc_comp_mct.F @@ -557,6 +557,13 @@ end subroutine xml_stream_get_attributes call check_clocks_sync(domain % clock, Eclock, err_tmp) ierr = ior(ierr,err_tmp) + ! Reset the alarm for checking for force setting of the adaptive timestep interval + if (mpas_is_alarm_ringing(domain % clock, 'adaptiveTimestepForceInterval', ierr=err_tmp)) then + ierr = ior(ierr, err_tmp) + call mpas_reset_clock_alarm(domain % clock, 'adaptiveTimestepForceInterval', ierr=err_tmp) + ierr = ior(ierr, err_tmp) + endif + ierr = ior(ierr, err_tmp) !----------------------------------------------------------------------- ! From 80c91e7ef3c03e43ed617116f8bcb48a4e96f4b0 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 31 Oct 2024 09:55:53 -0700 Subject: [PATCH 202/366] Enable MALI adaptive timestepper for mali-gis20km test --- .../testdefs/testmods_dirs/mali/gis20km/user_nl_mali | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/mpas-albany-landice/cime_config/testdefs/testmods_dirs/mali/gis20km/user_nl_mali b/components/mpas-albany-landice/cime_config/testdefs/testmods_dirs/mali/gis20km/user_nl_mali index 8b88e2add6c..4ef8179b7cf 100644 --- a/components/mpas-albany-landice/cime_config/testdefs/testmods_dirs/mali/gis20km/user_nl_mali +++ b/components/mpas-albany-landice/cime_config/testdefs/testmods_dirs/mali/gis20km/user_nl_mali @@ -1 +1,3 @@ config_am_globalstats_enable = .false. +config_adaptive_timestep = .true. +config_adaptive_timestep_force_interval = '0000-00-01_00:00:00' From 0e5ef8403e47a44a2200b8ef625d03d315c20604 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 31 Oct 2024 11:11:55 -0600 Subject: [PATCH 203/366] Refactor cmake --- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 118 +++++++++--------- 1 file changed, 57 insertions(+), 61 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 217c2945e48..d6793e6337b 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -44,75 +44,71 @@ else () set (FORCE_RUN_DIFF_FAILS "") endif() -# NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_ONLY_GENERATE_BASELINES) - CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" - LIBS p3 - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3;physics") - - # Make sure that a diff in the two implementation triggers a failed test (in debug only) - CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3 - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3;physics;fail" - ${FORCE_RUN_DIFF_FAILS}) - - if (NOT SCREAM_P3_SMALL_KERNELS) - CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" - LIBS p3_sk - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3_sk;physics") - - # Make sure that a diff in the two implementation triggers a failed test (in debug only) - CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3_sk - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3_sk;physics;fail" - ${FORCE_RUN_DIFF_FAILS}) - endif() -endif() - if (SCREAM_ENABLE_BASELINE_TESTS) if (SCREAM_ONLY_GENERATE_BASELINES) set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") else() - set(BASELINE_FILE_ARG "-b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + set(BASELINE_FILE_ARG "-c -b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") endif() +else() + set(BASELINE_FILE_ARG "--no-baselines") +endif() - CreateUnitTestExec(p3_run_and_cmp "p3_run_and_cmp.cpp" - LIBS p3 - EXCLUDE_MAIN_CPP) +CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" + LIBS p3 + EXE_ARGS "${BASELINE_FILE_ARG}" + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3;physics") - CreateUnitTestFromExec(p3_run_and_cmp_cxx p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "p3;physics") +# Make sure that a diff in the two implementation triggers a failed test (in debug only) +CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp + LIBS p3 + EXE_ARGS "${BASELINE_FILE_ARG}" + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3;physics;fail" + ${FORCE_RUN_DIFF_FAILS}) - CreateUnitTestFromExec(p3_run_and_cmp_f90 p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-f ${BASELINE_FILE_ARG}" - LABELS "p3;physics") +if (NOT SCREAM_P3_SMALL_KERNELS) + CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" + LIBS p3_sk + EXE_ARGS "${BASELINE_FILE_ARG}" + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3_sk;physics") - # Make sure that a diff from baselines triggers a failed test (in debug only) - CreateUnitTest(p3_run_and_cmp_cxx_fail "p3_run_and_cmp.cpp" - LIBS p3 - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "p3;physics;fail" - EXCLUDE_MAIN_CPP - ${FORCE_RUN_DIFF_FAILS}) + # Make sure that a diff in the two implementation triggers a failed test (in debug only) + CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp + LIBS p3_sk + EXE_ARGS "${BASELINE_FILE_ARG}" + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3_sk;physics;fail" + ${FORCE_RUN_DIFF_FAILS}) +endif() - # By default, baselines should be created using all fortran (ctest -L baseline_gen). If the user wants - # to use CXX to generate their baselines, they should use "ctest -L baseline_gen_cxx". - # Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it - if (SCREAM_TEST_MAX_THREADS GREATER 1) - # ECUT only adds _ompX if we have more than one value of X, or if X>1 - set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) - endif() - set_tests_properties (p3_run_and_cmp_f90${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;baseline_cmp") - set_tests_properties (p3_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") +CreateUnitTestExec(p3_run_and_cmp "p3_run_and_cmp.cpp" + LIBS p3 + EXCLUDE_MAIN_CPP) + +CreateUnitTestFromExec(p3_run_and_cmp_cxx p3_run_and_cmp + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "p3;physics") + +# Make sure that a diff from baselines triggers a failed test (in debug only) +CreateUnitTest(p3_run_and_cmp_cxx_fail "p3_run_and_cmp.cpp" + LIBS p3 + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "p3;physics;fail" + EXCLUDE_MAIN_CPP + ${FORCE_RUN_DIFF_FAILS}) + +# Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it +if (SCREAM_TEST_MAX_THREADS GREATER 1) + # ECUT only adds _ompX if we have more than one value of X, or if X>1 + set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) endif() + +set_tests_properties (p3_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") From 48ea3a94cce21acc0e9f570a7060533582f6255f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 31 Oct 2024 12:46:15 -0600 Subject: [PATCH 204/366] progress --- components/eamxx/src/physics/p3/p3_f90.cpp | 2 +- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 15 +++--- .../p3/tests/p3_rain_sed_unit_tests.cpp | 4 +- .../physics/p3/tests/p3_unit_tests_common.hpp | 49 ++++++++++++++++++- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_f90.cpp b/components/eamxx/src/physics/p3/p3_f90.cpp index 38bb84c416e..650ad3c645d 100644 --- a/components/eamxx/src/physics/p3/p3_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_f90.cpp @@ -79,7 +79,7 @@ void FortranDataIterator::init (const FortranData::Ptr& dp) { fdipb(nc); fdipb(qr); fdipb(nr); fdipb(qi); fdipb(ni); fdipb(qm); fdipb(bm); fdipb(precip_liq_surf); fdipb(precip_ice_surf); fdipb(diag_eff_radius_qc); fdipb(diag_eff_radius_qi); fdipb(diag_eff_radius_qr); fdipb(rho_qi); - fdipb(dpres); fdipb(inv_exner); fdipb(qv2qi_depos_tend); + fdipb(dpres); fdipb(inv_exner); fdipb(qv2qi_depos_tend); fdipb(precip_liq_flux); fdipb(precip_ice_flux); fdipb(cld_frac_r); fdipb(cld_frac_l); fdipb(cld_frac_i); fdipb(liq_ice_exchange); fdipb(vap_liq_exchange); diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index d6793e6337b..d5980d91cbf 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -44,26 +44,27 @@ else () set (FORCE_RUN_DIFF_FAILS "") endif() +# All tests should understand the same baseline args if (SCREAM_ENABLE_BASELINE_TESTS) if (SCREAM_ONLY_GENERATE_BASELINES) - set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data") else() - set(BASELINE_FILE_ARG "-c -b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + set(BASELINE_FILE_ARG "-c -b ${SCREAM_BASELINES_DIR}/data") endif() else() - set(BASELINE_FILE_ARG "--no-baselines") + set(BASELINE_FILE_ARG "-n") # no baselines endif() CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" LIBS p3 - EXE_ARGS "${BASELINE_FILE_ARG}" + EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3;physics") # Make sure that a diff in the two implementation triggers a failed test (in debug only) CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp LIBS p3 - EXE_ARGS "${BASELINE_FILE_ARG}" + EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3;physics;fail" @@ -72,14 +73,14 @@ CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp if (NOT SCREAM_P3_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" LIBS p3_sk - EXE_ARGS "${BASELINE_FILE_ARG}" + EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3_sk;physics") # Make sure that a diff in the two implementation triggers a failed test (in debug only) CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp LIBS p3_sk - EXE_ARGS "${BASELINE_FILE_ARG}" + EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3_sk;physics;fail" diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index 443b3801404..2d6cd8d0a00 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -20,7 +20,7 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestRainSed { +struct UnitWrap::UnitTest::TestRainSed : public UnitWrap::UnitTest::Base { static void run_phys_rain_vel() { @@ -219,7 +219,7 @@ TEST_CASE("p3_rain_sed", "[p3_functions]") { using TRS = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSed; - scream::p3::p3_init(); // need fortran table data + TRS::init(); TRS::run_phys(); TRS::run_bfb(); diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp index 68f4491f789..611173af2c1 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp @@ -3,6 +3,11 @@ #include "share/scream_types.hpp" #include "p3_functions.hpp" +#include "p3_f90.hpp" +#include "ekat/util/ekat_test_utils.hpp" + +#include +#include namespace scream { namespace p3 { @@ -20,6 +25,12 @@ namespace unit_test { struct UnitWrap { + enum BASELINE_ACTION { + NONE, + COMPARE, + GENERATE + }; + template struct UnitTest : public KokkosTypes { @@ -58,6 +69,43 @@ struct UnitWrap { static constexpr Int max_pack_size = 16; static constexpr Int num_test_itrs = max_pack_size / Spack::n; + struct Base { + static inline std::string BASELINE_PATH; + static inline BASELINE_ACTION ACTION; + + static void init() + { + scream::p3::p3_init(); // many tests will need fortran table data + auto& ts = ekat::TestSession::get(); + auto raw_flags = ts.flags.begin()->first; + std::stringstream ss(raw_flags); + std::string flag; + ACTION = NONE; + BASELINE_PATH = ""; + bool next_token_is_path = false; + while (ss >> flag) { + if (flag == "-c") { + ACTION = COMPARE; + } + else if (flag == "-g") { + ACTION = GENERATE; + } + else if (flag == "-n") { + ACTION = NONE; + } + else if (flag == "-b") { + next_token_is_path = true; + } + else if (next_token_is_path) { + BASELINE_PATH = flag; + next_token_is_path = false; + } + } + EKAT_REQUIRE_MSG( !(ACTION != NONE && BASELINE_PATH == ""), + "P3 unit test flags problem: baseline actions were requested but no baseline path was provided"); + } + }; + // Put struct decls here struct TestTableIce; struct TestTable3; @@ -102,7 +150,6 @@ struct UnitWrap { struct TestIceDepositionSublimation; struct TestPreventLiqSupersaturation; }; - }; } // namespace unit_test From 7f9aaacd31cf3f7bf20c1d3d8518c5725659669a Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 31 Oct 2024 14:17:24 -0600 Subject: [PATCH 205/366] Transition to object complete --- .../p3/tests/p3_autoconversion_unit_tests.cpp | 20 +++--- .../p3_back_to_cell_average_unit_tests.cpp | 15 ++-- ...lc_liq_relaxation_timescale_unit_tests.cpp | 13 ++-- .../tests/p3_calc_rime_density_unit_tests.cpp | 15 ++-- .../p3/tests/p3_check_values_unit_tests.cpp | 17 ++--- .../p3_cldliq_imm_freezing_unit_tests.cpp | 15 ++-- .../p3/tests/p3_cloud_rain_acc_unit_tests.cpp | 15 ++-- .../p3/tests/p3_cloud_sed_unit_tests.cpp | 15 ++-- .../tests/p3_droplet_self_coll_unit_tests.cpp | 15 ++-- .../physics/p3/tests/p3_dsd2_unit_tests.cpp | 24 ++++--- .../p3/tests/p3_evaporate_rain_unit_tests.cpp | 29 ++++---- .../physics/p3/tests/p3_find_unit_tests.cpp | 9 ++- .../tests/p3_get_latent_heat_unit_tests.cpp | 13 ++-- .../p3_ice_cldliq_wet_growth_unit_tests.cpp | 13 ++-- .../p3/tests/p3_ice_collection_unit_tests.cpp | 38 +++++----- .../p3_ice_deposition_sublimation_tests.cpp | 26 +++---- .../p3/tests/p3_ice_melting_unit_tests.cpp | 16 +++-- .../p3/tests/p3_ice_nucleation_unit_tests.cpp | 13 ++-- ...p3_ice_relaxation_timescale_unit_tests.cpp | 13 ++-- .../p3/tests/p3_ice_sed_unit_tests.cpp | 27 ++++---- .../p3_ice_supersat_conservation_tests.cpp | 9 +-- .../p3/tests/p3_ice_tables_unit_tests.cpp | 19 ++--- .../p3_incloud_mixingratios_unit_tests.cpp | 13 ++-- .../physics/p3/tests/p3_main_unit_tests.cpp | 31 ++++----- .../p3/tests/p3_nc_conservation_tests.cpp | 9 +-- .../p3/tests/p3_ni_conservation_tests.cpp | 9 +-- .../p3/tests/p3_nr_conservation_tests.cpp | 9 +-- .../p3_prevent_liq_supersaturation_tests.cpp | 18 +++-- .../tests/p3_rain_imm_freezing_unit_tests.cpp | 15 ++-- .../p3/tests/p3_rain_sed_unit_tests.cpp | 23 +++---- .../tests/p3_rain_self_collection_tests.cpp | 11 +-- ...p3_subgrid_variance_scaling_unit_tests.cpp | 22 +++--- .../physics/p3/tests/p3_table3_unit_tests.cpp | 9 +-- .../src/physics/p3/tests/p3_unit_tests.cpp | 69 +++++++++++-------- .../physics/p3/tests/p3_unit_tests_common.hpp | 26 ++++--- .../physics/p3/tests/p3_upwind_unit_tests.cpp | 26 +++---- 36 files changed, 360 insertions(+), 319 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp index 78f673121dd..e4ff1fd4366 100644 --- a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp @@ -19,10 +19,10 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3CloudWaterAutoconversion +struct UnitWrap::UnitTest::TestP3CloudWaterAutoconversion : public UnitWrap::UnitTest::Base { -static void cloud_water_autoconversion_unit_bfb_tests(){ +void cloud_water_autoconversion_unit_bfb_tests() { CloudWaterAutoconversionData cwadc[max_pack_size] = { // rho, qc_incld, nc_incld, inv_qc_relvar @@ -114,11 +114,11 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ } } - static void run_bfb(){ + void run_bfb() { cloud_water_autoconversion_unit_bfb_tests(); } - KOKKOS_FUNCTION static void autoconversion_is_positive(const Int &i, Int &errors){ + KOKKOS_FUNCTION static void autoconversion_is_positive(const Int &i, Int &errors){ const Spack rho(1.0), inv_qc_relvar(1.0); Spack qc_incld, nc_incld(1e7), qc2qr_autoconv_tend(0.0), nc2nr_autoconv_tend(0.0), ncautr(0.0); @@ -134,7 +134,7 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ } } - static void run_physics(){ + void run_physics(){ int nerr = 0; @@ -153,12 +153,14 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ } // namespace p3 } // namespace scream -namespace{ +namespace { TEST_CASE("p3_cloud_water_autoconversion_test", "[p3_cloud_water_autoconversion_test]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3CloudWaterAutoconversion::run_physics(); - scream::p3::unit_test::UnitWrap::UnitTest::TestP3CloudWaterAutoconversion::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3CloudWaterAutoconversion; + + T t; + t.run_physics(); + t.run_bfb(); } } // namespace - diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index 0377b990d78..f73bd3d993a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -19,14 +19,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestBackToCellAverage { +struct UnitWrap::UnitTest::TestBackToCellAverage : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { auto engine = setup_random_test(); @@ -181,12 +181,11 @@ namespace { TEST_CASE("p3_back_to_cell_average", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestBackToCellAverage; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestBackToCellAverage; - TRIF::run_phys(); - TRIF::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index 97193fd06d3..69715267176 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -20,14 +20,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale { +struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap::UnitTest::Base { - static void run_phys() + void run_phys() { // TODO } - static void run_bfb() + void run_bfb() { auto engine = setup_random_test(); @@ -115,10 +115,11 @@ namespace { TEST_CASE("p3_calc_liq_relaxation_timescale", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale; - TD::run_phys(); - TD::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp index 5d70c7302ae..f858797fa00 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCalcRimeDensity { +struct UnitWrap::UnitTest::TestCalcRimeDensity : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc cloud mixing ratio is large // enough to affect the rime density. @@ -144,12 +144,11 @@ namespace { TEST_CASE("p3_calc_rime_density", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcRimeDensity; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcRimeDensity; - TRIF::run_phys(); - TRIF::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp index f2f4a7f4404..8b9ecf3904f 100644 --- a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp @@ -20,9 +20,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCheckValues { +struct UnitWrap::UnitTest::TestCheckValues : public UnitWrap::UnitTest::Base { -static void run_check_values_bfb() +void run_check_values_bfb() { auto engine = setup_random_test(); @@ -60,7 +60,7 @@ static void run_check_values_bfb() } } -static void run_check_values_phys() +void run_check_values_phys() { // TODO } @@ -75,14 +75,11 @@ namespace { TEST_CASE("p3_check_values", "[p3_functions]") { - using TRS = scream::p3::unit_test::UnitWrap::UnitTest::TestCheckValues; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCheckValues; - scream::p3::p3_init(); // need fortran table data - - TRS::run_check_values_phys(); - TRS::run_check_values_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_check_values_phys(); + t.run_check_values_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp index eeccb70fd58..901650d8695 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCldliqImmersionFreezing { +struct UnitWrap::UnitTest::TestCldliqImmersionFreezing : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -127,12 +127,11 @@ namespace { TEST_CASE("p3_cldliq_immersion_freezing", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestCldliqImmersionFreezing; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCldliqImmersionFreezing; - TRIF::run_phys(); - TRIF::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp index 472f9de1987..2cfe22e5a26 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCloudRainAccretion { +struct UnitWrap::UnitTest::TestCloudRainAccretion : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -130,12 +130,11 @@ namespace { TEST_CASE("p3_cloud_rain_accretion", "[p3_functions]") { - using TCRA = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudRainAccretion; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudRainAccretion; - TCRA::run_phys(); - TCRA::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index 19fe5b3279e..b11400c98f4 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -19,14 +19,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCloudSed { +struct UnitWrap::UnitTest::TestCloudSed : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { auto engine = setup_random_test(); @@ -98,12 +98,11 @@ namespace { TEST_CASE("p3_cloud_sed", "[p3_functions]") { - using TCS = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudSed; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudSed; - TCS::run_phys(); - TCS::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp index b81505816ea..fccf41db627 100644 --- a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestDropletSelfCollection { +struct UnitWrap::UnitTest::TestDropletSelfCollection : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -124,12 +124,11 @@ namespace { TEST_CASE("p3_droplet_self_collection", "[p3_functions]") { - using TCRA = scream::p3::unit_test::UnitWrap::UnitTest::TestDropletSelfCollection; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestDropletSelfCollection; - TCRA::run_phys(); - TCRA::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 50894b03855..43c5a7de8dd 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -22,9 +22,9 @@ namespace unit_test { */ template -struct UnitWrap::UnitTest::TestDsd2 { +struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { - static void run_cloud_bfb() + void run_cloud_bfb() { // Read in tables view_2d_table vn_table_vals; view_2d_table vm_table_vals; view_2d_table revap_table_vals; @@ -107,12 +107,12 @@ struct UnitWrap::UnitTest::TestDsd2 { } } - static void run_cloud_phys() + void run_cloud_phys() { // TODO } - static void run_rain_bfb() + void run_rain_bfb() { using KTH = KokkosTypes; @@ -190,7 +190,7 @@ struct UnitWrap::UnitTest::TestDsd2 { } } - static void run_rain_phys() + void run_rain_phys() { // TODO } @@ -204,18 +204,20 @@ namespace { TEST_CASE("p3_cloud_dsd2", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; - TD::run_cloud_phys(); - TD::run_cloud_bfb(); + T t; + t.run_cloud_phys(); + t.run_cloud_bfb(); } TEST_CASE("p3_rain_dsd2", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; - TD::run_rain_phys(); - TD::run_rain_bfb(); + T t; + t.run_rain_phys(); + t.run_rain_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index f8398135a19..9df3ecdb0c8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -8,21 +8,14 @@ #include "p3_unit_tests_common.hpp" -//#include -//#include -//#include -//#include -//#include // std::setprecision - namespace scream { namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestEvapSublPrecip -{ +struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest::Base { - static void run_property(){ + void run_property() { //TEST WEIGHTING TIMESCALE //======================== @@ -133,9 +126,9 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip REQUIRE( qrtend[0] <= qr_incld[0]/dt); REQUIRE( nrtend[0] <= nr_incld[0]/dt); //keep end-of-step nr positive. Should always be true. - }; //end run_property + } //end run_property - static void run_bfb(){ + void run_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; @@ -267,14 +260,18 @@ namespace { TEST_CASE("p3_evaporate_rain_property", "p3_unit_tests") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; - TestStruct::run_property(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; + + T t; + t.run_property(); } TEST_CASE("p3_evaporate_rain_test", "p3_unit_tests") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; - TestStruct::run_bfb(); -} + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; + + T t; + t.run_bfb(); + } }// end anonymous namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp index 4ba2dc6760e..9c812f8f842 100644 --- a/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp @@ -22,9 +22,9 @@ namespace unit_test { // template -struct UnitWrap::UnitTest::TestFind { +struct UnitWrap::UnitTest::TestFind : public UnitWrap::UnitTest::Base { -static void run() +void run() { const int max_threads = #ifdef KOKKOS_ENABLE_OPENMP @@ -112,7 +112,10 @@ namespace { TEST_CASE("p3_find", "[p3_functions]") { - scream::p3::unit_test::UnitWrap::UnitTest::TestFind::run(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestFind; + + T t; + t.run(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp index 5d7b4de4ea7..dfe6f3ca6a4 100644 --- a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp @@ -20,9 +20,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestLatentHeat { +struct UnitWrap::UnitTest::TestLatentHeat : public UnitWrap::UnitTest::Base { - static void run_latent_heat_bfb() + void run_latent_heat_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; @@ -55,7 +55,7 @@ struct UnitWrap::UnitTest::TestLatentHeat { } } - static void run_latent_heat_phys() + void run_latent_heat_phys() { // TODO } @@ -69,10 +69,11 @@ namespace { TEST_CASE("p3_latent_heat", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestLatentHeat; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestLatentHeat; - TD::run_latent_heat_phys(); - TD::run_latent_heat_bfb(); + T t; + t.run_latent_heat_phys(); + t.run_latent_heat_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index caf6638d5cd..e2a61b37481 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { +struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest::Base { - static void run_ice_cldliq_wet_growth_bfb() + void run_ice_cldliq_wet_growth_bfb() { using KTH = KokkosTypes; @@ -127,7 +127,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { } } - static void run_ice_cldliq_wet_growth_phys() + void run_ice_cldliq_wet_growth_phys() { // TODO } @@ -141,10 +141,11 @@ namespace { TEST_CASE("p3_ice_cldliq_wet_growth", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCldliqWetGrowth; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCldliqWetGrowth; - TD::run_ice_cldliq_wet_growth_phys(); - TD::run_ice_cldliq_wet_growth_bfb(); + T t; + t.run_ice_cldliq_wet_growth_phys(); + t.run_ice_cldliq_wet_growth_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index 8fa1649b121..9c628c0641c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -22,9 +22,9 @@ namespace unit_test { * Unit-tests for p3 ice collection functions. */ template -struct UnitWrap::UnitTest::TestIceCollection { +struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest::Base { - static void run_ice_cldliq_bfb() + void run_ice_cldliq_bfb() { // Read in tables view_2d_table vn_table_vals; @@ -119,12 +119,12 @@ struct UnitWrap::UnitTest::TestIceCollection { } } - static void run_ice_cldliq_phys() + void run_ice_cldliq_phys() { // TODO } - static void run_ice_rain_bfb() + void run_ice_rain_bfb() { using KTH = KokkosTypes; @@ -206,12 +206,12 @@ struct UnitWrap::UnitTest::TestIceCollection { } } - static void run_ice_rain_phys() + void run_ice_rain_phys() { // TODO } - static void run_ice_self_bfb() + void run_ice_self_bfb() { using KTH = KokkosTypes; @@ -283,8 +283,7 @@ struct UnitWrap::UnitTest::TestIceCollection { } } - - static void run_ice_self_phys() + void run_ice_self_phys() { // TODO } @@ -298,24 +297,29 @@ namespace { TEST_CASE("p3_ice_cldliq", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; - TD::run_ice_cldliq_phys(); - TD::run_ice_cldliq_bfb(); + T t; + t.run_ice_cldliq_phys(); + t.run_ice_cldliq_bfb(); } TEST_CASE("p3_ice_rain", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; - TD::run_ice_rain_phys(); - TD::run_ice_rain_bfb(); + T t; + t.run_ice_rain_phys(); + t.run_ice_rain_bfb(); } TEST_CASE("p3_ice_self", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; - TD::run_ice_self_phys(); - TD::run_ice_self_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; + + T t; + t.run_ice_self_phys(); + t.run_ice_self_bfb(); } + } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp index bc1921441bb..f6afd915dc6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp @@ -13,9 +13,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceDepositionSublimation { +struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::UnitTest::Base { - static void run_property(){ + void run_property() { //Note that a lot of property tests are included in run_bfb for simplicity //Choose default values (just grabbed a row from the array in run_bfb) @@ -30,13 +30,13 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { //init output vars Spack qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend; - + //CHECK THAT UNREASONABLY LARGE VAPOR DEPOSITION DOESN'T LEAVE QV SUBSATURATED WRT QI Spack epsi_tmp=1e6; //make 1/(sat removal timescale) huge so vapdep rate removes all supersat in 1 dt. Functions::ice_deposition_sublimation(qi_incld, ni_incld, T_atm, qv_sat_l, qv_sat_i, epsi_tmp, abi, qv, inv_dt, qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend); REQUIRE( (qv2qi_vapdep_tend[0]==0 || std::abs( qv2qi_vapdep_tend[0] - (qv[0] - qv_sat_i[0])*inv_dt) <1e-8) ); - + //CHECK THAT HUGE SUBLIMATION DOESN'T LEAVE QV SUPERSATURATED WRT QI Spack qv_sat_i_tmp=1e-2; Functions::ice_deposition_sublimation(qi_incld, ni_incld, T_atm, qv_sat_l, qv_sat_i_tmp, @@ -46,8 +46,8 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { //CHECK BEHAVIOR AS DT->0? } - - static void run_bfb() + + void run_bfb() { IceDepositionSublimationData f90_data[max_pack_size] = { {1.0000E-04,4.5010E+05,2.8750E+02,1.1279E-02,1.1279E-02,0.0000E+00,3.3648E+00,5.0000E-03,1.666667e-02}, @@ -67,7 +67,7 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { {5.1000E-03,5.1317E+05,2.5834E+02,1.6757E-03,1.4491E-03,6.0620E-02,1.3763E+00,5.0000E-03,1.666667e-02}, {5.0000E-08,5.4479E+05,2.4793E+02,7.5430E-04,5.8895E-04,4.6769E-04,1.1661E+00,1.5278E-04,1.666667e-02}, }; - + static constexpr Int num_runs = sizeof(f90_data) / sizeof(IceDepositionSublimationData); // Generate random input data @@ -162,16 +162,18 @@ namespace { TEST_CASE("ice_deposition_sublimation_property", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; - TestStruct::run_property(); + T t; + t.run_property(); } - + TEST_CASE("ice_deposition_sublimation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index 28c6582f8a1..5e29579a910 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -19,10 +19,10 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3IceMelting +struct UnitWrap::UnitTest::TestP3IceMelting : public UnitWrap::UnitTest::Base { -static void ice_melting_bfb(){ +void ice_melting_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; @@ -105,7 +105,7 @@ static void ice_melting_bfb(){ REQUIRE(IceMelt[s].ni2nr_melt_tend == IceMelt_host(s).ni2nr_melt_tend); } } -}; // TestP3IceMelting +} }; // UnitWrap @@ -113,11 +113,13 @@ static void ice_melting_bfb(){ } // namespace p3 } // namespace scream -namespace{ +namespace { + +TEST_CASE("p3_ice_melting_test", "[p3_ice_melting_test]") { + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3IceMelting; -TEST_CASE("p3_ice_melting_test", "[p3_ice_melting_test]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3IceMelting::ice_melting_bfb(); + T t; + t.ice_melting_bfb(); } } // namespace - diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp index 4e54b8837e0..51ab5031c45 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceNucleation { +struct UnitWrap::UnitTest::TestIceNucleation : public UnitWrap::UnitTest::Base { - static void run_ice_nucleation_bfb() + void run_ice_nucleation_bfb() { using KTH = KokkosTypes; @@ -104,7 +104,7 @@ struct UnitWrap::UnitTest::TestIceNucleation { } //end for do_prescribed_CCN } - static void run_ice_nucleation_phys() + void run_ice_nucleation_phys() { // TODO } @@ -118,10 +118,11 @@ namespace { TEST_CASE("p3_ice_nucleation", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceNucleation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceNucleation; - TD::run_ice_nucleation_phys(); - TD::run_ice_nucleation_bfb(); + T t; + t.run_ice_nucleation_phys(); + t.run_ice_nucleation_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp index a75b55fdbec..dfb668bc81d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceRelaxationTimescale { +struct UnitWrap::UnitTest::TestIceRelaxationTimescale : public UnitWrap::UnitTest::Base { - static void run_ice_relaxation_timescale_bfb() + void run_ice_relaxation_timescale_bfb() { using KTH = KokkosTypes; @@ -101,7 +101,7 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale { } } - static void run_ice_relaxation_timescale_phys() + void run_ice_relaxation_timescale_phys() { // TODO } @@ -115,10 +115,11 @@ namespace { TEST_CASE("p3_ice_relaxation_timescale", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceRelaxationTimescale; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceRelaxationTimescale; - TD::run_ice_relaxation_timescale_phys(); - TD::run_ice_relaxation_timescale_bfb(); + T t; + t.run_ice_relaxation_timescale_phys(); + t.run_ice_relaxation_timescale_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index c04aa55be00..86ff76e9169 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -19,31 +19,31 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceSed { +struct UnitWrap::UnitTest::TestIceSed : public UnitWrap::UnitTest::Base { -static void run_phys_calc_bulk_rhime() +void run_phys_calc_bulk_rhime() { // TODO } -static void run_phys_ice_sed() +void run_phys_ice_sed() { // TODO } -static void run_phys_homogeneous_freezing() +void run_phys_homogeneous_freezing() { // TODO } -static void run_phys() +void run_phys() { run_phys_calc_bulk_rhime(); run_phys_ice_sed(); run_phys_homogeneous_freezing(); } -static void run_bfb_calc_bulk_rhime() +void run_bfb_calc_bulk_rhime() { constexpr Scalar qsmall = C::QSMALL; @@ -123,7 +123,7 @@ static void run_bfb_calc_bulk_rhime() } } -static void run_bfb_ice_sed() +void run_bfb_ice_sed() { auto engine = setup_random_test(); @@ -187,7 +187,7 @@ static void run_bfb_ice_sed() } } -static void run_bfb_homogeneous_freezing() +void run_bfb_homogeneous_freezing() { constexpr Scalar latice = C::LatIce; @@ -256,7 +256,7 @@ static void run_bfb_homogeneous_freezing() } } -static void run_bfb() +void run_bfb() { run_bfb_calc_bulk_rhime(); run_bfb_ice_sed(); @@ -273,12 +273,11 @@ namespace { TEST_CASE("p3_ice_sed", "[p3_functions]") { - using TCS = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSed; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSed; - TCS::run_phys(); - TCS::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp index 5873b8335c3..3c8121cdf3c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp @@ -14,9 +14,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceSupersatConservation { +struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; @@ -98,9 +98,10 @@ namespace { TEST_CASE("ice_supersat_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSupersatConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSupersatConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp index 091d05895fc..0b171b0a7ff 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp @@ -22,9 +22,9 @@ namespace unit_test { */ template -struct UnitWrap::UnitTest::TestTableIce { +struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base { - static void test_read_lookup_tables_bfb() + void test_read_lookup_tables_bfb() { // Read in ice tables view_ice_table ice_table_vals; @@ -62,7 +62,7 @@ struct UnitWrap::UnitTest::TestTableIce { } template - static void init_table_linear_dimension(View& table, int linear_dimension) + void init_table_linear_dimension(View& table, int linear_dimension) { // set up views using NonConstView = typename View::non_const_type; @@ -96,7 +96,7 @@ struct UnitWrap::UnitTest::TestTableIce { table = view_device; } - static void run_bfb() + void run_bfb() { using KTH = KokkosTypes; @@ -294,7 +294,7 @@ struct UnitWrap::UnitTest::TestTableIce { } } - static void run_phys() + void run_phys() { #if 0 view_ice_table ice_table_vals; @@ -343,11 +343,12 @@ namespace { TEST_CASE("p3_ice_tables", "[p3_functions]") { - using TTI = scream::p3::unit_test::UnitWrap::UnitTest::TestTableIce; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestTableIce; - TTI::test_read_lookup_tables_bfb(); - TTI::run_phys(); - TTI::run_bfb(); + T t; + t.test_read_lookup_tables_bfb(); + t.run_phys(); + t.run_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp index 4a405fcc7c4..a75bdbd1728 100644 --- a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIncloudMixing { +struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest::Base { - static void run_incloud_mixing_bfb() + void run_incloud_mixing_bfb() { using KTH = KokkosTypes; @@ -129,7 +129,7 @@ struct UnitWrap::UnitTest::TestIncloudMixing { } } - static void run_incloud_mixing_phys() + void run_incloud_mixing_phys() { // TODO } @@ -143,10 +143,11 @@ namespace { TEST_CASE("p3_incloud_mixingratios", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIncloudMixing; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIncloudMixing; - TD::run_incloud_mixing_phys(); - TD::run_incloud_mixing_bfb(); + T t; + t.run_incloud_mixing_phys(); + t.run_incloud_mixing_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index a804bd8756d..35089c0e7f9 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -19,29 +19,29 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3Main { +struct UnitWrap::UnitTest::TestP3Main : public UnitWrap::UnitTest::Base { -static void run_phys_p3_main_part1() +void run_phys_p3_main_part1() { // TODO } -static void run_phys_p3_main_part2() +void run_phys_p3_main_part2() { // TODO } -static void run_phys_p3_main_part3() +void run_phys_p3_main_part3() { // TODO } -static void run_phys_p3_main() +void run_phys_p3_main() { // TODO } -static void run_phys() +void run_phys() { run_phys_p3_main_part1(); run_phys_p3_main_part2(); @@ -49,7 +49,7 @@ static void run_phys() run_phys_p3_main(); } -static void run_bfb_p3_main_part1() +void run_bfb_p3_main_part1() { auto engine = setup_random_test(); @@ -150,7 +150,7 @@ static void run_bfb_p3_main_part1() } } -static void run_bfb_p3_main_part2() +void run_bfb_p3_main_part2() { auto engine = setup_random_test(); @@ -274,7 +274,7 @@ static void run_bfb_p3_main_part2() } } -static void run_bfb_p3_main_part3() +void run_bfb_p3_main_part3() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; @@ -369,7 +369,7 @@ static void run_bfb_p3_main_part3() } } -static void run_bfb_p3_main() +void run_bfb_p3_main() { auto engine = setup_random_test(); @@ -476,7 +476,7 @@ static void run_bfb_p3_main() } } -static void run_bfb() +void run_bfb() { run_bfb_p3_main_part1(); run_bfb_p3_main_part2(); @@ -494,12 +494,11 @@ namespace { TEST_CASE("p3_main", "[p3_functions]") { - using TP3 = scream::p3::unit_test::UnitWrap::UnitTest::TestP3Main; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3Main; - TP3::run_phys(); - TP3::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp index ffe7ea504e1..ecf47b99eaa 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp @@ -14,9 +14,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestNcConservation { +struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { auto engine = setup_random_test(); @@ -93,9 +93,10 @@ namespace { TEST_CASE("nc_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestNcConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestNcConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp index 3aa3428825c..41c13afd923 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp @@ -14,9 +14,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestNiConservation { +struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { auto engine = setup_random_test(); @@ -92,9 +92,10 @@ namespace { TEST_CASE("ni_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestNiConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestNiConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp index fb9e1e758b4..2df8a01dbb3 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp @@ -14,9 +14,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestNrConservation { +struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { auto engine = setup_random_test(); @@ -97,9 +97,10 @@ namespace { TEST_CASE("nr_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestNrConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestNrConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index a1b27056281..2b845e9c97b 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -15,9 +15,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { +struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() //Conceptual tests for prevent_liq_supersaturation. Note many conceptual tests make sense to run on //random data, so are included in run_bfb rather than here. { @@ -88,7 +88,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { } //end run_property - static void run_bfb() + void run_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; @@ -187,14 +187,18 @@ namespace { TEST_CASE("prevent_liq_supersaturation_property", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; - TestStruct::run_property(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; + + T t; + t.run_property(); } TEST_CASE("prevent_liq_supersaturation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; - TestStruct::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; + + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp index ae142ae77ac..e19b83d6c2a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestRainImmersionFreezing { +struct UnitWrap::UnitTest::TestRainImmersionFreezing : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -124,12 +124,11 @@ namespace { TEST_CASE("p3_rain_immersion_freezing", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestRainImmersionFreezing; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestRainImmersionFreezing; - TRIF::run_phys(); - TRIF::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index 2d6cd8d0a00..8184d5df0f3 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -22,23 +22,23 @@ namespace unit_test { template struct UnitWrap::UnitTest::TestRainSed : public UnitWrap::UnitTest::Base { -static void run_phys_rain_vel() +void run_phys_rain_vel() { // TODO } -static void run_phys_rain_sed() +void run_phys_rain_sed() { // TODO } -static void run_phys() +void run_phys() { run_phys_rain_vel(); run_phys_rain_sed(); } -static void run_bfb_rain_vel() +void run_bfb_rain_vel() { // Read in tables view_2d_table vn_table_vals; view_2d_table vm_table_vals; view_2d_table revap_table_vals; @@ -124,7 +124,7 @@ static void run_bfb_rain_vel() } } -static void run_bfb_rain_sed() +void run_bfb_rain_sed() { auto engine = setup_random_test(); @@ -201,7 +201,7 @@ static void run_bfb_rain_sed() } } -static void run_bfb() +void run_bfb() { run_bfb_rain_vel(); run_bfb_rain_sed(); @@ -217,14 +217,11 @@ namespace { TEST_CASE("p3_rain_sed", "[p3_functions]") { - using TRS = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSed; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSed; - TRS::init(); - - TRS::run_phys(); - TRS::run_bfb(); - - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp index a8c0dbc07dd..7bcdf59dac8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp @@ -22,9 +22,9 @@ namespace unit_test { * Unit-tests for p3 ice collection functions. */ template -struct UnitWrap::UnitTest::TestRainSelfCollection { +struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest::Base { - static void run_rain_self_collection_bfb_tests(){ + void run_rain_self_collection_bfb_tests() { RainSelfCollectionData dc[max_pack_size] = { // rho, qr_incld, nr_incld, nr_selfcollect_tend @@ -101,7 +101,7 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { } } - static void run_bfb(){ + void run_bfb() { run_rain_self_collection_bfb_tests(); } @@ -114,7 +114,10 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { namespace { TEST_CASE("p3_rain_self_collection_test", "[p3_rain_self_collection_test"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestRainSelfCollection::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSelfCollection; + + T t; + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp index 9a728b6c57e..9e75889df0c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp @@ -19,11 +19,11 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling +struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::UnitTest::Base { //----------------------------------------------------------------- - static void run_bfb_tests(){ + void run_bfb_tests() { //test that C++ and F90 implementations are BFB //Set of relvar values to loop over @@ -81,7 +81,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling //----------------------------------------------------------------- KOKKOS_FUNCTION static void subgrid_variance_scaling_linearity_test(const Scalar& relvar, - int& errors){ + int& errors) { //If expon=1, subgrid_variance_scaling should be 1 Scalar tol = C::macheps * 1e3; //1e3 is scale factor to make pass, essentially an estimate of numerical error @@ -97,7 +97,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } //----------------------------------------------------------------- - KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar1_test(int& errors){ + KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar1_test(int& errors) { //If relvar=1, subgrid_variance_scaling should be factorial(expon) Scalar tol = C::macheps * 1e3; //1e3 is scale factor to make pass, essentially an estimate of numerical error @@ -116,7 +116,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } //----------------------------------------------------------------- - KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar3_test(int& errors){ + KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar3_test(int& errors) { //If expon=3, subgrid variance scaling should be relvar^3+3*relvar^2+2*relvar/relvar^3 Scalar tol = C::macheps * 100; //100 is a fudge factor to make sure tests pass. 10 was too small for gnu on CPU. @@ -151,7 +151,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } //end relvar3_test //----------------------------------------------------------------- - static void run_property_tests(){ + void run_property_tests() { /*This function executes all the SGS variance scaling tests by looping *over a bunch of test and summing their return statuses. *If that sum is zero, no errors have occurred. Otherwise you have errors. @@ -189,12 +189,14 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } // namespace p3 } // namespace scream -namespace{ +namespace { TEST_CASE("p3_subgrid_variance_scaling_test", "[p3_subgrid_variance_scaling_test]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3SubgridVarianceScaling::run_bfb_tests(); - scream::p3::unit_test::UnitWrap::UnitTest::TestP3SubgridVarianceScaling::run_property_tests(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3SubgridVarianceScaling; + + T t; + t.run_bfb_tests(); + t.run_property_tests(); } } // namespace - diff --git a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp index c99d156004e..cc14bfc46d2 100644 --- a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp @@ -44,7 +44,7 @@ namespace unit_test { // refinement, where the mesh is a 1D mesh transecting the table domain. template -struct UnitWrap::UnitTest::TestTable3 { +struct UnitWrap::UnitTest::TestTable3 : public UnitWrap::UnitTest::Base { KOKKOS_FUNCTION static Scalar calc_lamr (const Scalar& mu_r, const Scalar& alpha) { // Parameters for lower and upper bounds, derived above, multiplied by @@ -68,7 +68,7 @@ struct UnitWrap::UnitTest::TestTable3 { return Functions::apply_table(table, t3); } - static void run () { + void run () { // This test doesn't use mu_r_table_vals, as that is not a table3 type. It // doesn't matter whether we use vm_table_vals or vn_table_vals, as the table values // don't matter in what we are testing; we are testing interpolation @@ -170,9 +170,10 @@ namespace { TEST_CASE("p3_tables", "[p3_functions]") { - scream::p3::p3_init(); // need fortran table data + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestTable3; - scream::p3::unit_test::UnitWrap::UnitTest::TestTable3::run(); + T t; + t.run(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index 3b994cfdad5..2a33cc70db1 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -21,10 +21,10 @@ namespace unit_test { * Unit-tests for p3_functions. */ template -struct UnitWrap::UnitTest::TestP3Conservation +struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest::Base { - static void cloud_water_conservation_tests_device() { + void cloud_water_conservation_tests_device() { using KTH = KokkosTypes; @@ -78,7 +78,7 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE(cwdc_host[0].qc2qr_autoconv_tend * cwdc[0].dt <= cwdc_host[0].qc); } - static void rain_water_conservation_tests_device() { + void rain_water_conservation_tests_device() { using KTH = KokkosTypes; RainWaterConservationData rwdc[1] = {{sp(1e-5), 0.0, 0.0, 0.0, 0.0, sp(1.1), sp(1e-4), 0.0, 0.0 }}; @@ -131,7 +131,7 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE( rwdc_host(0).qr2qv_evap_tend * rwdc_host(0).dt <= rwdc_host(0).qr); } - static void ice_water_conservation_tests_device(){ + void ice_water_conservation_tests_device() { using KTH = KokkosTypes; IceWaterConservationData iwdc[1] = {{sp(1e-5), 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, sp(1.1), sp(1e-4), 0.0}}; @@ -173,7 +173,7 @@ struct UnitWrap::UnitTest::TestP3Conservation } - static void run() + void run() { cloud_water_conservation_tests_device(); @@ -182,7 +182,7 @@ struct UnitWrap::UnitTest::TestP3Conservation ice_water_conservation_tests_device(); } - static void cloud_water_conservation_unit_bfb_tests(){ + void cloud_water_conservation_unit_bfb_tests() { using KTH = KokkosTypes; @@ -277,7 +277,7 @@ struct UnitWrap::UnitTest::TestP3Conservation } } - static void ice_water_conservation_unit_bfb_tests() + void ice_water_conservation_unit_bfb_tests() { using KTH = KokkosTypes; @@ -372,7 +372,7 @@ struct UnitWrap::UnitTest::TestP3Conservation } } - static void rain_water_conservation_unit_bfb_tests(){ + void rain_water_conservation_unit_bfb_tests() { using KTH = KokkosTypes; @@ -462,7 +462,7 @@ struct UnitWrap::UnitTest::TestP3Conservation } } - static void run_bfb() { + void run_bfb() { cloud_water_conservation_unit_bfb_tests(); rain_water_conservation_unit_bfb_tests(); @@ -473,9 +473,9 @@ struct UnitWrap::UnitTest::TestP3Conservation }; template -struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce +struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitTest::Base { - static void update_prognostic_ice_unit_bfb_tests() { + void update_prognostic_ice_unit_bfb_tests() { constexpr Scalar nmltratio = C::nmltratio; constexpr Scalar dt = 1.8000E+03; @@ -695,16 +695,16 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce } } - static void run_bfb(){ + void run_bfb() { update_prognostic_ice_unit_bfb_tests(); } }; //TestP3UpdatePrognosticIce template -struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables +struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::UnitTest::Base { - static void get_time_space_phys_variables_unit_bfb_tests(){ + void get_time_space_phys_variables_unit_bfb_tests() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; @@ -809,15 +809,15 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables } } - static void run_bfb(){ + void run_bfb() { get_time_space_phys_variables_unit_bfb_tests(); } }; //TestGetTimeSpacePhysVariables template -struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq +struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitTest::Base { - static void update_prognostic_liquid_unit_bfb_tests(){ + void update_prognostic_liquid_unit_bfb_tests() { constexpr Scalar latvap = C::LatVap; //fortran generated data is input to the following @@ -983,16 +983,16 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq } } - static void run_bfb(){ + void run_bfb() { update_prognostic_liquid_unit_bfb_tests(); } }; //TestP3UpdatePrognosticLiq template -struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi +struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi : public UnitWrap::UnitTest::Base { - static void impose_max_total_ni_bfb_test(){ + void impose_max_total_ni_bfb_test() { constexpr Scalar max_total_ni = 740.0e3; ImposeMaxTotalNiData dc[max_pack_size]= { @@ -1062,7 +1062,7 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi } } - static void run_bfb(){ + void run_bfb() { impose_max_total_ni_bfb_test(); } @@ -1075,24 +1075,39 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi namespace { TEST_CASE("p3_conservation_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3Conservation::run(); - scream::p3::unit_test::UnitWrap::UnitTest::TestP3Conservation::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3Conservation; + + T t; + t.run(); + t.run_bfb(); } TEST_CASE("p3_get_time_space_phys_variables_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestGetTimeSpacePhysVariables::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestGetTimeSpacePhysVariables; + + T t; + t.run_bfb(); } TEST_CASE("p3_update_prognostic_ice_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticIce::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticIce; + + T t; + t.run_bfb(); } TEST_CASE("p3_update_prognostic_liquid_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticLiq::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticLiq; + + T t; + t.run_bfb(); } TEST_CASE("p3_impose_max_total_ni_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi; + + T t; + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp index 611173af2c1..685284a0463 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp @@ -5,6 +5,7 @@ #include "p3_functions.hpp" #include "p3_f90.hpp" #include "ekat/util/ekat_test_utils.hpp" +#include "p3_functions_f90.hpp" #include #include @@ -70,40 +71,45 @@ struct UnitWrap { static constexpr Int num_test_itrs = max_pack_size / Spack::n; struct Base { - static inline std::string BASELINE_PATH; - static inline BASELINE_ACTION ACTION; + std::string m_baseline_path; + BASELINE_ACTION m_baseline_action; - static void init() + Base() : + m_baseline_path(""), + m_baseline_action(NONE) { scream::p3::p3_init(); // many tests will need fortran table data auto& ts = ekat::TestSession::get(); auto raw_flags = ts.flags.begin()->first; std::stringstream ss(raw_flags); std::string flag; - ACTION = NONE; - BASELINE_PATH = ""; bool next_token_is_path = false; while (ss >> flag) { if (flag == "-c") { - ACTION = COMPARE; + m_baseline_action = COMPARE; } else if (flag == "-g") { - ACTION = GENERATE; + m_baseline_action = GENERATE; } else if (flag == "-n") { - ACTION = NONE; + m_baseline_action = NONE; } else if (flag == "-b") { next_token_is_path = true; } else if (next_token_is_path) { - BASELINE_PATH = flag; + m_baseline_path = flag; next_token_is_path = false; } } - EKAT_REQUIRE_MSG( !(ACTION != NONE && BASELINE_PATH == ""), + EKAT_REQUIRE_MSG( !(m_baseline_action != NONE && m_baseline_path == ""), "P3 unit test flags problem: baseline actions were requested but no baseline path was provided"); } + + ~Base() + { + scream::p3::P3GlobalForFortran::deinit(); + } }; // Put struct decls here diff --git a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp index fde2ca644cb..b24a0ebb4a2 100644 --- a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp @@ -36,9 +36,9 @@ namespace unit_test { // cells in the domain are 0. This lets us check the restricted-domain usage of // the upwind routine in the first time step. template -struct UnitWrap::UnitTest::TestUpwind { +struct UnitWrap::UnitTest::TestUpwind : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { using ekat::repack; constexpr auto SPS = SCREAM_SMALL_PACK_SIZE; @@ -201,7 +201,7 @@ static void run_phys() } } -static void run_bfb() +void run_bfb() { auto engine = setup_random_test(); @@ -274,14 +274,14 @@ static void run_bfb() }; template -struct UnitWrap::UnitTest::TestGenSed { +struct UnitWrap::UnitTest::TestGenSed : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { auto engine = setup_random_test(); @@ -358,18 +358,20 @@ namespace { TEST_CASE("p3_upwind", "[p3_functions]") { - using TU = scream::p3::unit_test::UnitWrap::UnitTest::TestUpwind; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestUpwind; - TU::run_phys(); - TU::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } TEST_CASE("p3_gen_sed", "[p3_functions]") { - using TG = scream::p3::unit_test::UnitWrap::UnitTest::TestGenSed; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestGenSed; - TG::run_phys(); - TG::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace From 4d404f9b0eeb07850a75068dbafef3b347e38bde Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 31 Oct 2024 16:15:23 -0600 Subject: [PATCH 206/366] rain_sed working --- .../eamxx/src/physics/p3/p3_functions_f90.hpp | 14 +++++ .../p3/tests/p3_rain_sed_unit_tests.cpp | 59 +++++++++++++------ .../src/physics/share/physics_test_data.cpp | 15 +++++ .../src/physics/share/physics_test_data.hpp | 39 +++++++++++- 4 files changed, 107 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 89e6ac56908..364a068a423 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -4,6 +4,7 @@ #include "physics/p3/p3_functions.hpp" #include "physics/share/physics_test_data.hpp" #include "share/scream_types.hpp" +#include "ekat/util/ekat_file_utils.hpp" #include #include @@ -471,6 +472,19 @@ struct ComputeRainFallVelocityData // Outputs Real mu_r, lamr, V_qr, V_nr; + + PTD_RW_SCALARS(5, nr_incld, mu_r, lamr, V_qr, V_nr); + + void read(const ekat::FILEPtr& fid) + { + read_scalars(fid); + } + + void write(const ekat::FILEPtr& fid) const + { + write_scalars(fid); + } + }; /////////////////////////////////////////////////////////////////////////////// diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index 8184d5df0f3..22f0c50b41d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -46,7 +46,7 @@ void run_bfb_rain_vel() Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Load some lookup inputs, need at least one per pack value - ComputeRainFallVelocityData crfv_fortran[max_pack_size] = { + ComputeRainFallVelocityData crfv_baseline[max_pack_size] = { // qr_incld, rhofacr, nr_incld {1.1030E-04, 1.3221E+00, 6.2964E+05}, {2.1437E-13, 1.0918E+00, 6.5337E+07}, @@ -70,16 +70,20 @@ void run_bfb_rain_vel() }; - // Sync to device, needs to happen before fortran calls so that + // Sync to device, needs to happen before reads so that // inout data is in original state view_1d crfv_device("crfv", max_pack_size); const auto crfv_host = Kokkos::create_mirror_view(crfv_device); - std::copy(&crfv_fortran[0], &crfv_fortran[0] + max_pack_size, crfv_host.data()); + std::copy(&crfv_baseline[0], &crfv_baseline[0] + max_pack_size, crfv_host.data()); Kokkos::deep_copy(crfv_device, crfv_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - compute_rain_fall_velocity(crfv_fortran[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/rain_fall_velocity.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + crfv_baseline[i].read(fid); + } } // Calc bulk rime from a kernel and copy results back to host @@ -112,21 +116,28 @@ void run_bfb_rain_vel() // Sync back to host Kokkos::deep_copy(crfv_host, crfv_device); - // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + // Validate results for (Int s = 0; s < max_pack_size; ++s) { - REQUIRE(crfv_fortran[s].nr_incld == crfv_host(s).nr_incld); - REQUIRE(crfv_fortran[s].mu_r == crfv_host(s).mu_r); - REQUIRE(crfv_fortran[s].lamr == crfv_host(s).lamr); - REQUIRE(crfv_fortran[s].V_qr == crfv_host(s).V_qr); - REQUIRE(crfv_fortran[s].V_nr == crfv_host(s).V_nr); + REQUIRE(crfv_baseline[s].nr_incld == crfv_host(s).nr_incld); + REQUIRE(crfv_baseline[s].mu_r == crfv_host(s).mu_r); + REQUIRE(crfv_baseline[s].lamr == crfv_host(s).lamr); + REQUIRE(crfv_baseline[s].V_qr == crfv_host(s).V_qr); + REQUIRE(crfv_baseline[s].V_nr == crfv_host(s).V_nr); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + crfv_host(s).write(fid); } } } void run_bfb_rain_sed() { - auto engine = setup_random_test(); + // With stored baselines, we must use a fixed seed! + auto engine = setup_random_test(1267351); // F90 is quite slow on weaver, so we decrease dt to reduce // the number of steps in rain_sed. @@ -151,7 +162,7 @@ void run_bfb_rain_sed() d.randomize(engine, { {d.qr_incld, {C::QSMALL/2, C::QSMALL*2}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state RainSedData rsds_cxx[num_runs] = { RainSedData(rsds_fortran[0]), @@ -160,9 +171,13 @@ void run_bfb_rain_sed() RainSedData(rsds_fortran[3]), }; - // Get data from fortran - for (auto& d : rsds_fortran) { - rain_sedimentation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/rain_sed.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (auto& d : rsds_fortran) { + d.read(fid); + } } // Get data from cxx @@ -180,7 +195,7 @@ void run_bfb_rain_sed() d.qr_tend, d.nr_tend); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space Int start = std::min(rsds_fortran[i].kbot, rsds_fortran[i].ktop) - 1; // 0-based indx @@ -199,6 +214,12 @@ void run_bfb_rain_sed() REQUIRE(rsds_fortran[i].precip_liq_surf == rsds_cxx[i].precip_liq_surf); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + rsds_cxx[i].write(fid); + } + } } void run_bfb() diff --git a/components/eamxx/src/physics/share/physics_test_data.cpp b/components/eamxx/src/physics/share/physics_test_data.cpp index 4a13528e98b..986a7c0800f 100644 --- a/components/eamxx/src/physics/share/physics_test_data.cpp +++ b/components/eamxx/src/physics/share/physics_test_data.cpp @@ -26,4 +26,19 @@ PhysicsTestData& PhysicsTestData::assignment_impl(const PhysicsTestData& rhs) return *this; } +void PhysicsTestData::read(const ekat::FILEPtr& fid) +{ + m_reals.read(fid); + m_ints.read(fid); + m_bools.read(fid); +} + +void PhysicsTestData::write(const ekat::FILEPtr& fid) const +{ + m_reals.write(fid); + m_ints.write(fid); + m_bools.write(fid); +} + + } // namespace scream diff --git a/components/eamxx/src/physics/share/physics_test_data.hpp b/components/eamxx/src/physics/share/physics_test_data.hpp index 099480b4439..fca5ca04cc0 100644 --- a/components/eamxx/src/physics/share/physics_test_data.hpp +++ b/components/eamxx/src/physics/share/physics_test_data.hpp @@ -5,6 +5,7 @@ #include "ekat/util/ekat_math_utils.hpp" #include "ekat/ekat_assert.hpp" +#include "ekat/util/ekat_file_utils.hpp" #include #include @@ -90,12 +91,34 @@ struct SHOCGridData : public PhysicsTestData { #define PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s ) PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) ; s = rhs.s #define PTD_ASS20(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t ) PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) ; t = rhs.t +#define PTD_RW0(action ) ((void) (0)) +#define PTD_RW1(action, a ) ekat::action(&a, 1, fid) +#define PTD_RW2(action, a, b ) PTD_RW1(action, a) ; ekat::action(&b, 1, fid) +#define PTD_RW3(action, a, b, c ) PTD_RW2(action, a, b) ; ekat::action(&c, 1, fid) +#define PTD_RW4(action, a, b, c, d ) PTD_RW3(action, a, b, c) ; ekat::action(&d, 1, fid) +#define PTD_RW5(action, a, b, c, d, e ) PTD_RW4(action, a, b, c, d) ; ekat::action(&e, 1, fid) +#define PTD_RW6(action, a, b, c, d, e, f ) PTD_RW5(action, a, b, c, d, e) ; ekat::action(&f, 1, fid) +#define PTD_RW7(action, a, b, c, d, e, f, g ) PTD_RW6(action, a, b, c, d, e, f) ; ekat::action(&g, 1, fid) +#define PTD_RW8(action, a, b, c, d, e, f, g, h ) PTD_RW7(action, a, b, c, d, e, f, g) ; ekat::action(&h, 1, fid) +#define PTD_RW9(action, a, b, c, d, e, f, g, h, i ) PTD_RW8(action, a, b, c, d, e, f, g, h) ; ekat::action(&i, 1, fid) +#define PTD_RW10(action, a, b, c, d, e, f, g, h, i, j ) PTD_RW9(action, a, b, c, d, e, f, g, h, i) ; ekat::action(&j, 1, fid) + #define PTD_ASSIGN_OP(name, num_scalars, ...) \ name& operator=(const name& rhs) { PTD_ASS##num_scalars(__VA_ARGS__); assignment_impl(rhs); return *this; } +#define PTD_RW_SCALARS(num_scalars, ...) \ + void read_scalars(const ekat::FILEPtr& fid) { PTD_RW##num_scalars(read, __VA_ARGS__); } \ + void write_scalars(const ekat::FILEPtr& fid) const { PTD_RW##num_scalars(write, __VA_ARGS__); } + +#define PTD_RW() \ + void read(const ekat::FILEPtr& fid) { read_scalars(fid); PhysicsTestData::read(fid); } \ + void write(const ekat::FILEPtr& fid) const { write_scalars(fid); PhysicsTestData::write(fid); } + #define PTD_STD_DEF(name, num_scalars, ...) \ PTD_DATA_COPY_CTOR(name, num_scalars); \ - PTD_ASSIGN_OP(name, num_scalars, __VA_ARGS__) + PTD_ASSIGN_OP(name, num_scalars, __VA_ARGS__) \ + PTD_RW() \ + PTD_RW_SCALARS(num_scalars, __VA_ARGS__) namespace scream { @@ -241,6 +264,16 @@ class PhysicsTestData m_data = new_data; } + void read(const ekat::FILEPtr& fid) + { + ekat::read(m_data.data(), m_data.size(), fid); + } + + void write(const ekat::FILEPtr& fid) const + { + ekat::write(m_data.data(), m_data.size(), fid); + } + std::vector > m_dims_list; // list of dims, one per unique set of dims std::vector > m_members_list; // list of member pointers, same outer index space as m_dims_list std::vector m_data; // the member data in a flat vector @@ -336,6 +369,10 @@ class PhysicsTestData } } + void read(const ekat::FILEPtr& fid); + + void write(const ekat::FILEPtr& fid) const; + protected: PhysicsTestData& assignment_impl(const PhysicsTestData& rhs); From 40d5114cc7ebae278cf299f852d2580a8889b4c7 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 29 Oct 2024 20:20:46 -0500 Subject: [PATCH 207/366] Fix total land-ice freshwater flux in data mode Previously, the total was only being computed when thermodynamics below ice shelves are actively computed, whereas we need to compute the total of the interface flux and the frazil flux when the interface flux comes from a data file as well. While we expect the frazil flux to be zero, these code modifications do not assume or require this to be true. --- .../shared/mpas_ocn_surface_land_ice_fluxes.F | 287 +++++++++--------- 1 file changed, 145 insertions(+), 142 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F index c249f202757..fe489029a13 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F @@ -492,168 +492,179 @@ subroutine ocn_surface_land_ice_fluxes_build_arrays(meshPool, & err = 0 - if (.not.landIceStandaloneOn) return + if (.not. (landIceStandaloneOn .or. landIceDataOn)) return call mpas_timer_start("land_ice_build_arrays") call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_array(forcingPool, 'landIcePressure', landIcePressure) - - call mpas_pool_get_array(forcingPool, 'landIceFloatingFraction', landIceFloatingFraction) - call mpas_pool_get_array(forcingPool, 'landIceFloatingMask', landIceFloatingMask) - call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) - call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) - call mpas_pool_get_array(forcingPool, 'heatFluxToLandIce', heatFluxToLandIce) - call mpas_pool_get_array(forcingPool, 'frazilIceFreshwaterFlux', frazilIceFreshwaterFlux) call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFluxTotal', landIceFreshwaterFluxTotal) - - call mpas_pool_get_array(forcingPool, 'landIceInterfaceTracers', landIceInterfaceTracers) - call mpas_pool_get_dimension(forcingPool, & - 'index_landIceInterfaceTemperature', & - indexITPtr) - call mpas_pool_get_dimension(forcingPool, & - 'index_landIceInterfaceSalinity', & - indexISPtr) - indexIT = indexITPtr - indexIS = indexISPtr - - if (useHollandJenkinsAdvDiff) then - call mpas_pool_get_array(forcingPool, 'landIceSurfaceTemperature', landIceSurfaceTemperature) - - allocate(freezeInterfaceSalinity(nCells), & - freezeInterfaceTemperature(nCells), & - freezeFreshwaterFlux(nCells), & - freezeHeatFlux(nCells), & - freezeIceHeatFlux(nCells)) - end if + call mpas_pool_get_array(forcingPool, 'landIceFloatingMask', landIceFloatingMask) nCells = nCellsArray( size(nCellsArray) ) - if (isomipOn) then !*** ISOMIP formulation + if (landIceStandaloneOn) then - !$omp parallel - !$omp do schedule(runtime) private(freshwaterFlux, heatFlux) - do iCell = 1, nCells - if (landIceFloatingMask(iCell) == 0) cycle + call mpas_pool_get_array(forcingPool, 'landIcePressure', landIcePressure) + call mpas_pool_get_array(forcingPool, 'landIceFloatingFraction', landIceFloatingFraction) + call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) + call mpas_pool_get_array(forcingPool, 'heatFluxToLandIce', heatFluxToLandIce) + + + call mpas_pool_get_array(forcingPool, 'landIceInterfaceTracers', landIceInterfaceTracers) + call mpas_pool_get_dimension(forcingPool, & + 'index_landIceInterfaceTemperature', & + indexITPtr) + call mpas_pool_get_dimension(forcingPool, & + 'index_landIceInterfaceSalinity', & + indexISPtr) + indexIT = indexITPtr + indexIS = indexISPtr + + if (useHollandJenkinsAdvDiff) then + call mpas_pool_get_array(forcingPool, 'landIceSurfaceTemperature', landIceSurfaceTemperature) + + allocate(freezeInterfaceSalinity(nCells), & + freezeInterfaceTemperature(nCells), & + freezeFreshwaterFlux(nCells), & + freezeHeatFlux(nCells), & + freezeIceHeatFlux(nCells)) + end if - ! linearized equaiton for the S and p dependent potential freezing temperature - landIceInterfaceTracers(indexIT,iCell) = ocn_freezing_temperature( & - salinity=landIceBoundaryLayerTracers(indexBLT,iCell), & - pressure=landIcePressure(iCell), & - inLandIceCavity=.true.) + if (isomipOn) then !*** ISOMIP formulation - ! using (3) and (4) from Hunter (2006) - ! or (7) from Jenkins et al. (2001) if gamma constant - ! and no heat flux into ice - ! freshwater flux = density * melt rate is in kg/m^2/s - freshwaterFlux = -rho_sw * ISOMIPgammaT * (cp_sw/latent_heat_fusion_mks) & - * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell)) + !$omp parallel + !$omp do schedule(runtime) private(freshwaterFlux, heatFlux) + do iCell = 1, nCells + if (landIceFloatingMask(iCell) == 0) cycle - landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*freshwaterFlux + ! linearized equaiton for the S and p dependent potential freezing temperature + landIceInterfaceTracers(indexIT,iCell) = ocn_freezing_temperature( & + salinity=landIceBoundaryLayerTracers(indexBLT,iCell), & + pressure=landIcePressure(iCell), & + inLandIceCavity=.true.) - ! Using (13) from Jenkins et al. (2001) - ! heat flux is in W/s - heatFlux = cp_sw*(freshwaterFlux*landIceInterfaceTracers(indexIT,iCell) & - + rho_sw*ISOMIPgammaT & - * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell))) - landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*heatFlux + ! using (3) and (4) from Hunter (2006) + ! or (7) from Jenkins et al. (2001) if gamma constant + ! and no heat flux into ice + ! freshwater flux = density * melt rate is in kg/m^2/s + freshwaterFlux = -rho_sw * ISOMIPgammaT * (cp_sw/latent_heat_fusion_mks) & + * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell)) - heatFluxToLandIce(iCell) = 0.0_RKIND + landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*freshwaterFlux - end do - !$omp end do - !$omp end parallel - endif ! isomipOn + ! Using (13) from Jenkins et al. (2001) + ! heat flux is in W/s + heatFlux = cp_sw*(freshwaterFlux*landIceInterfaceTracers(indexIT,iCell) & + + rho_sw*ISOMIPgammaT & + * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell))) + landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*heatFlux - if (jenkinsOn .or. hollandJenkinsOn) then - if(useHollandJenkinsAdvDiff) then - ! melting solution - call compute_HJ99_melt_fluxes( & - landIceFloatingMask, & - landIceBoundaryLayerTracers(indexBLT,:), & - landIceBoundaryLayerTracers(indexBLS,:), & - landIceTracerTransferVelocities(indexHeatTrans,:), & - landIceTracerTransferVelocities(indexSaltTrans,:), & - landIceSurfaceTemperature, & - landIcePressure, & - landIceInterfaceTracers(indexIT,:), & - landIceInterfaceTracers(indexIS,:), & - landIceFreshwaterFlux, & - landIceHeatFlux, & - heatFluxToLandIce, & - nCells, & - err) - if(err .ne. 0) then - call mpas_log_write( & - 'compute_HJ99_melt_fluxes failed.', & - MPAS_LOG_CRIT) - end if + heatFluxToLandIce(iCell) = 0.0_RKIND - ! freezing solution - call compute_melt_fluxes( & - landIceFloatingMask, & - landIceBoundaryLayerTracers(indexBLT,:), & - landIceBoundaryLayerTracers(indexBLS,:), & - landIceTracerTransferVelocities(indexHeatTrans,:), & - landIceTracerTransferVelocities(indexSaltTrans,:), & - landIcePressure, & - freezeInterfaceTemperature, & - freezeInterfaceSalinity, & - freezeFreshwaterFlux, & - freezeHeatFlux, & - freezeIceHeatFlux, & - nCells, & - err) - if(err .ne. 0) then - call mpas_log_write( & - 'compute_melt_fluxes failed.', & - MPAS_LOG_CRIT) + end do + !$omp end do + !$omp end parallel + end if ! isomipOn + + if (jenkinsOn .or. hollandJenkinsOn) then + if(useHollandJenkinsAdvDiff) then + ! melting solution + call compute_HJ99_melt_fluxes( & + landIceFloatingMask, & + landIceBoundaryLayerTracers(indexBLT,:), & + landIceBoundaryLayerTracers(indexBLS,:), & + landIceTracerTransferVelocities(indexHeatTrans,:), & + landIceTracerTransferVelocities(indexSaltTrans,:), & + landIceSurfaceTemperature, & + landIcePressure, & + landIceInterfaceTracers(indexIT,:), & + landIceInterfaceTracers(indexIS,:), & + landIceFreshwaterFlux, & + landIceHeatFlux, & + heatFluxToLandIce, & + nCells, & + err) + if(err .ne. 0) then + call mpas_log_write( & + 'compute_HJ99_melt_fluxes failed.', & + MPAS_LOG_CRIT) + end if + + ! freezing solution + call compute_melt_fluxes( & + landIceFloatingMask, & + landIceBoundaryLayerTracers(indexBLT,:), & + landIceBoundaryLayerTracers(indexBLS,:), & + landIceTracerTransferVelocities(indexHeatTrans,:), & + landIceTracerTransferVelocities(indexSaltTrans,:), & + landIcePressure, & + freezeInterfaceTemperature, & + freezeInterfaceSalinity, & + freezeFreshwaterFlux, & + freezeHeatFlux, & + freezeIceHeatFlux, & + nCells, & + err) + if(err .ne. 0) then + call mpas_log_write( & + 'compute_melt_fluxes failed.', & + MPAS_LOG_CRIT) + end if + + do iCell = 1, nCells + if ((landIceFloatingMask(iCell) == 0) .or. (landIceFreshwaterFlux(iCell) >= 0.0_RKIND)) cycle + + landIceInterfaceTracers(indexIS,iCell) = freezeInterfaceSalinity(iCell) + landIceInterfaceTracers(indexIT,iCell) = freezeInterfaceTemperature(iCell) + landIceFreshwaterFlux(iCell) = freezeFreshwaterFlux(iCell) + landIceHeatFlux(iCell) = freezeHeatFlux(iCell) + heatFluxToLandIce(iCell) = freezeIceHeatFlux(iCell) + end do + else ! not using Holland and Jenkins advection/diffusion + call compute_melt_fluxes( & + landIceFloatingMask, & + landIceBoundaryLayerTracers(indexBLT,:), & + landIceBoundaryLayerTracers(indexBLS,:), & + landIceTracerTransferVelocities(indexHeatTrans,:), & + landIceTracerTransferVelocities(indexSaltTrans,:), & + landIcePressure, & + landIceInterfaceTracers(indexIT,:), & + landIceInterfaceTracers(indexIS,:), & + landIceFreshwaterFlux, & + landIceHeatFlux, & + heatFluxToLandIce, & + nCells, & + err) + if(err .ne. 0) then + call mpas_log_write( & + 'compute_melt_fluxes failed.', & + MPAS_LOG_CRIT) + end if end if + ! modulate the fluxes by the landIceFloatingFraction do iCell = 1, nCells - if ((landIceFloatingMask(iCell) == 0) .or. (landIceFreshwaterFlux(iCell) >= 0.0_RKIND)) cycle + if (landIceFloatingMask(iCell) == 0) cycle - landIceInterfaceTracers(indexIS,iCell) = freezeInterfaceSalinity(iCell) - landIceInterfaceTracers(indexIT,iCell) = freezeInterfaceTemperature(iCell) - landIceFreshwaterFlux(iCell) = freezeFreshwaterFlux(iCell) - landIceHeatFlux(iCell) = freezeHeatFlux(iCell) - heatFluxToLandIce(iCell) = freezeIceHeatFlux(iCell) + landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*landIceFreshwaterFlux(iCell) + landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*landIceHeatFlux(iCell) + heatFluxToLandIce(iCell) = landIceFloatingFraction(iCell)*heatFluxToLandIce(iCell) end do - else ! not using Holland and Jenkins advection/diffusion - call compute_melt_fluxes( & - landIceFloatingMask, & - landIceBoundaryLayerTracers(indexBLT,:), & - landIceBoundaryLayerTracers(indexBLS,:), & - landIceTracerTransferVelocities(indexHeatTrans,:), & - landIceTracerTransferVelocities(indexSaltTrans,:), & - landIcePressure, & - landIceInterfaceTracers(indexIT,:), & - landIceInterfaceTracers(indexIS,:), & - landIceFreshwaterFlux, & - landIceHeatFlux, & - heatFluxToLandIce, & - nCells, & - err) - if(err .ne. 0) then - call mpas_log_write( & - 'compute_melt_fluxes failed.', & - MPAS_LOG_CRIT) - end if - end if - ! modulate the fluxes by the landIceFloatingFraction - do iCell = 1, nCells - if (landIceFloatingMask(iCell) == 0) cycle + end if ! jenkinsOn or hollandJenkinsOn - landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*landIceFreshwaterFlux(iCell) - landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*landIceHeatFlux(iCell) - heatFluxToLandIce(iCell) = landIceFloatingFraction(iCell)*heatFluxToLandIce(iCell) - end do + if(useHollandJenkinsAdvDiff) then + deallocate(freezeInterfaceSalinity, & + freezeInterfaceTemperature, & + freezeFreshwaterFlux, & + freezeHeatFlux, & + freezeIceHeatFlux) + end if - endif ! jenkinsOn or hollandJenkinsOn + end if ! landIceStandaloneOn ! Add frazil and interface melt/freeze to get total fresh water flux if ( associated(frazilIceFreshwaterFlux) ) then @@ -666,14 +677,6 @@ subroutine ocn_surface_land_ice_fluxes_build_arrays(meshPool, & end do end if - if(useHollandJenkinsAdvDiff) then - deallocate(freezeInterfaceSalinity, & - freezeInterfaceTemperature, & - freezeFreshwaterFlux, & - freezeHeatFlux, & - freezeIceHeatFlux) - end if - call mpas_timer_stop("land_ice_build_arrays") !-------------------------------------------------------------------- From f669e8beb2e9a88942bdb2cc7a1e59ea2f705a75 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 11:35:20 -0600 Subject: [PATCH 208/366] Upwind working --- .../eamxx/src/physics/p3/p3_functions_f90.hpp | 2 + .../physics/p3/tests/p3_upwind_unit_tests.cpp | 102 +++++++++++------- .../src/physics/share/physics_test_data.cpp | 2 + .../src/physics/share/physics_test_data.hpp | 57 ++++++---- 4 files changed, 100 insertions(+), 63 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 364a068a423..20207460c0a 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -353,6 +353,8 @@ struct GenSedData : public CalcUpwindData PTD_DATA_COPY_CTOR(GenSedData, 10); PTD_ASSIGN_OP(GenSedData, 11, kts, kte, kdir, kbot, k_qxtop, num_arrays, dt_sub, Co_max, k_qxbot, dt_left, prt_accum); + PTD_RW(); + PTD_RW_SCALARS(11, kts, kte, kdir, kbot, k_qxtop, num_arrays, dt_sub, Co_max, k_qxbot, dt_left, prt_accum); }; /////////////////////////////////////////////////////////////////////////////// diff --git a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp index b24a0ebb4a2..b6b5307966a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp @@ -203,9 +203,10 @@ void run_phys() void run_bfb() { - auto engine = setup_random_test(); + // With stored baselines, we must use a fixed seed! + auto engine = setup_random_test(12345745); - CalcUpwindData cuds_fortran[] = { + CalcUpwindData cuds_baseline[] = { // kts, kte, kdir, kbot, k_qxtop, na, dt_sub, CalcUpwindData( 1, 72, -1, 72, 36, 2, 1.833E+03), CalcUpwindData( 1, 72, 1, 36, 72, 2, 1.833E+03), @@ -216,28 +217,32 @@ void run_bfb() CalcUpwindData( 1, 32, -1, 21, 7, 1, 1.833E+03), }; - static constexpr Int num_runs = sizeof(cuds_fortran) / sizeof(CalcUpwindData); + static constexpr Int num_runs = sizeof(cuds_baseline) / sizeof(CalcUpwindData); // Set up random input data - for (auto& d : cuds_fortran) { + for (auto& d : cuds_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CalcUpwindData cuds_cxx[num_runs] = { - CalcUpwindData(cuds_fortran[0]), - CalcUpwindData(cuds_fortran[1]), - CalcUpwindData(cuds_fortran[2]), - CalcUpwindData(cuds_fortran[3]), - CalcUpwindData(cuds_fortran[4]), - CalcUpwindData(cuds_fortran[5]), - CalcUpwindData(cuds_fortran[6]), + CalcUpwindData(cuds_baseline[0]), + CalcUpwindData(cuds_baseline[1]), + CalcUpwindData(cuds_baseline[2]), + CalcUpwindData(cuds_baseline[3]), + CalcUpwindData(cuds_baseline[4]), + CalcUpwindData(cuds_baseline[5]), + CalcUpwindData(cuds_baseline[6]), }; - // Get data from fortran - for (auto& d : cuds_fortran) { - calc_first_order_upwind_step(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/upwind.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (auto& d : cuds_baseline) { + d.read(fid); + } } // Get data from cxx @@ -251,17 +256,17 @@ void run_bfb() d.num_arrays, fluxes, vs, qnx); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(cuds_fortran[i].kbot, cuds_fortran[i].k_qxtop) - 1; // 0-based indx - Int end = std::max(cuds_fortran[i].kbot, cuds_fortran[i].k_qxtop); // 0-based indx + Int start = std::min(cuds_baseline[i].kbot, cuds_baseline[i].k_qxtop) - 1; // 0-based indx + Int end = std::max(cuds_baseline[i].kbot, cuds_baseline[i].k_qxtop); // 0-based indx Real** fluxesf90, **vsf90, **qnxf90, **fluxescxx, **vscxx, **qnxcxx; - cuds_fortran[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); + cuds_baseline[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); cuds_cxx[i].convert_to_ptr_arr(tmp1, fluxescxx, vscxx, qnxcxx); - for (int n = 0; n < cuds_fortran[i].num_arrays; ++n) { + for (int n = 0; n < cuds_baseline[i].num_arrays; ++n) { for (Int k = start; k < end; ++k) { REQUIRE(fluxesf90[n][k] == fluxescxx[n][k]); REQUIRE(qnxf90[n][k] == qnxcxx[n][k]); @@ -269,6 +274,12 @@ void run_bfb() } } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + cuds_cxx[i].write(fid); + } + } } }; @@ -283,9 +294,10 @@ void run_phys() void run_bfb() { - auto engine = setup_random_test(); + // With stored baselines, we must use a fixed seed! + auto engine = setup_random_test(2346563); - GenSedData gsds_fortran[] = { + GenSedData gsds_baseline[] = { // kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, num_arrays GenSedData(1, 72, -1, 36, 72, 72, 9.196E-02, 1.818E+01, 4.959E-05, 2), GenSedData(1, 72, -1, 36, 57, 72, 4.196E-01, 1.418E+02, 4.959E-06, 1), @@ -293,25 +305,29 @@ void run_bfb() GenSedData(1, 72, -1, 72, 72, 72, 4.196E-01, 1.418E+02, 4.959E-06, 1), }; - static constexpr Int num_runs = sizeof(gsds_fortran) / sizeof(GenSedData); + static constexpr Int num_runs = sizeof(gsds_baseline) / sizeof(GenSedData); // Set up random input data - for (auto& d : gsds_fortran) { + for (auto& d : gsds_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state GenSedData gsds_cxx[num_runs] = { - GenSedData(gsds_fortran[0]), - GenSedData(gsds_fortran[1]), - GenSedData(gsds_fortran[2]), - GenSedData(gsds_fortran[3]), + GenSedData(gsds_baseline[0]), + GenSedData(gsds_baseline[1]), + GenSedData(gsds_baseline[2]), + GenSedData(gsds_baseline[3]), }; - // Get data from fortran - for (auto& d : gsds_fortran) { - generalized_sedimentation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/gen_sed.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (auto& d : gsds_baseline) { + d.read(fid); + } } // Get data from cxx @@ -325,25 +341,31 @@ void run_bfb() d.num_arrays, fluxes, vs, qnx); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(gsds_fortran[i].k_qxbot, gsds_fortran[i].k_qxtop) - 1; // 0-based indx - Int end = std::max(gsds_fortran[i].k_qxbot, gsds_fortran[i].k_qxtop); // 0-based indx + Int start = std::min(gsds_baseline[i].k_qxbot, gsds_baseline[i].k_qxtop) - 1; // 0-based indx + Int end = std::max(gsds_baseline[i].k_qxbot, gsds_baseline[i].k_qxtop); // 0-based indx Real** fluxesf90, **vsf90, **qnxf90, **fluxescxx, **vscxx, **qnxcxx; - gsds_fortran[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); + gsds_baseline[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); gsds_cxx[i].convert_to_ptr_arr(tmp1, fluxescxx, vscxx, qnxcxx); - for (int n = 0; n < gsds_fortran[i].num_arrays; ++n) { + for (int n = 0; n < gsds_baseline[i].num_arrays; ++n) { for (Int k = start; k < end; ++k) { REQUIRE(fluxesf90[n][k] == fluxescxx[n][k]); REQUIRE(qnxf90[n][k] == qnxcxx[n][k]); } } - REQUIRE(gsds_fortran[i].k_qxbot == gsds_cxx[i].k_qxbot); - REQUIRE(gsds_fortran[i].dt_left == gsds_cxx[i].dt_left); - REQUIRE(gsds_fortran[i].prt_accum == gsds_cxx[i].prt_accum); + REQUIRE(gsds_baseline[i].k_qxbot == gsds_cxx[i].k_qxbot); + REQUIRE(gsds_baseline[i].dt_left == gsds_cxx[i].dt_left); + REQUIRE(gsds_baseline[i].prt_accum == gsds_cxx[i].prt_accum); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + gsds_cxx[i].write(fid); } } } diff --git a/components/eamxx/src/physics/share/physics_test_data.cpp b/components/eamxx/src/physics/share/physics_test_data.cpp index 986a7c0800f..f6c6d733fd0 100644 --- a/components/eamxx/src/physics/share/physics_test_data.cpp +++ b/components/eamxx/src/physics/share/physics_test_data.cpp @@ -28,6 +28,8 @@ PhysicsTestData& PhysicsTestData::assignment_impl(const PhysicsTestData& rhs) void PhysicsTestData::read(const ekat::FILEPtr& fid) { + EKAT_REQUIRE_MSG(fid, + "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); m_reals.read(fid); m_ints.read(fid); m_bools.read(fid); diff --git a/components/eamxx/src/physics/share/physics_test_data.hpp b/components/eamxx/src/physics/share/physics_test_data.hpp index fca5ca04cc0..153ec7b9b28 100644 --- a/components/eamxx/src/physics/share/physics_test_data.hpp +++ b/components/eamxx/src/physics/share/physics_test_data.hpp @@ -69,17 +69,17 @@ struct SHOCGridData : public PhysicsTestData { #define PTD_DATA_COPY_CTOR(name, num_args) \ name(const name& rhs) : name(PTD_ONES(num_args)) { *this = rhs; } -#define PTD_ASS0( ) ((void) (0)) -#define PTD_ASS1(a ) a = rhs.a -#define PTD_ASS2(a, b ) PTD_ASS1(a) ; b = rhs.b -#define PTD_ASS3(a, b, c ) PTD_ASS2(a, b) ; c = rhs.c -#define PTD_ASS4(a, b, c, d ) PTD_ASS3(a, b, c) ; d = rhs.d -#define PTD_ASS5(a, b, c, d, e ) PTD_ASS4(a, b, c, d) ; e = rhs.e -#define PTD_ASS6(a, b, c, d, e, f ) PTD_ASS5(a, b, c, d, e) ; f = rhs.f -#define PTD_ASS7(a, b, c, d, e, f, g ) PTD_ASS6(a, b, c, d, e, f) ; g = rhs.g -#define PTD_ASS8(a, b, c, d, e, f, g, h ) PTD_ASS7(a, b, c, d, e, f, g) ; h = rhs.h -#define PTD_ASS9(a, b, c, d, e, f, g, h, i ) PTD_ASS8(a, b, c, d, e, f, g, h) ; i = rhs.i -#define PTD_ASS10(a, b, c, d, e, f, g, h, i, j ) PTD_ASS9(a, b, c, d, e, f, g, h, i) ; j = rhs.j +#define PTD_ASS0( ) ((void) (0)) +#define PTD_ASS1(a ) PTD_ASS0() ; a = rhs.a +#define PTD_ASS2(a, b ) PTD_ASS1(a) ; b = rhs.b +#define PTD_ASS3(a, b, c ) PTD_ASS2(a, b) ; c = rhs.c +#define PTD_ASS4(a, b, c, d ) PTD_ASS3(a, b, c) ; d = rhs.d +#define PTD_ASS5(a, b, c, d, e ) PTD_ASS4(a, b, c, d) ; e = rhs.e +#define PTD_ASS6(a, b, c, d, e, f ) PTD_ASS5(a, b, c, d, e) ; f = rhs.f +#define PTD_ASS7(a, b, c, d, e, f, g ) PTD_ASS6(a, b, c, d, e, f) ; g = rhs.g +#define PTD_ASS8(a, b, c, d, e, f, g, h ) PTD_ASS7(a, b, c, d, e, f, g) ; h = rhs.h +#define PTD_ASS9(a, b, c, d, e, f, g, h, i ) PTD_ASS8(a, b, c, d, e, f, g, h) ; i = rhs.i +#define PTD_ASS10(a, b, c, d, e, f, g, h, i, j ) PTD_ASS9(a, b, c, d, e, f, g, h, i) ; j = rhs.j #define PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k ) PTD_ASS10(a, b, c, d, e, f, g, h, i, j) ; k = rhs.k #define PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l ) PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k) ; l = rhs.l #define PTD_ASS13(a, b, c, d, e, f, g, h, i, j, k, l, m ) PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l) ; m = rhs.m @@ -91,23 +91,34 @@ struct SHOCGridData : public PhysicsTestData { #define PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s ) PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) ; s = rhs.s #define PTD_ASS20(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t ) PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) ; t = rhs.t -#define PTD_RW0(action ) ((void) (0)) -#define PTD_RW1(action, a ) ekat::action(&a, 1, fid) -#define PTD_RW2(action, a, b ) PTD_RW1(action, a) ; ekat::action(&b, 1, fid) -#define PTD_RW3(action, a, b, c ) PTD_RW2(action, a, b) ; ekat::action(&c, 1, fid) -#define PTD_RW4(action, a, b, c, d ) PTD_RW3(action, a, b, c) ; ekat::action(&d, 1, fid) -#define PTD_RW5(action, a, b, c, d, e ) PTD_RW4(action, a, b, c, d) ; ekat::action(&e, 1, fid) -#define PTD_RW6(action, a, b, c, d, e, f ) PTD_RW5(action, a, b, c, d, e) ; ekat::action(&f, 1, fid) -#define PTD_RW7(action, a, b, c, d, e, f, g ) PTD_RW6(action, a, b, c, d, e, f) ; ekat::action(&g, 1, fid) -#define PTD_RW8(action, a, b, c, d, e, f, g, h ) PTD_RW7(action, a, b, c, d, e, f, g) ; ekat::action(&h, 1, fid) -#define PTD_RW9(action, a, b, c, d, e, f, g, h, i ) PTD_RW8(action, a, b, c, d, e, f, g, h) ; ekat::action(&i, 1, fid) -#define PTD_RW10(action, a, b, c, d, e, f, g, h, i, j ) PTD_RW9(action, a, b, c, d, e, f, g, h, i) ; ekat::action(&j, 1, fid) + +#define PTD_RW0(action ) ((void) (0)) +#define PTD_RW1(action, a ) PTD_RW0(action) ; ekat::action(&a, 1, fid) +#define PTD_RW2(action, a, b ) PTD_RW1(action, a) ; ekat::action(&b, 1, fid) +#define PTD_RW3(action, a, b, c ) PTD_RW2(action, a, b) ; ekat::action(&c, 1, fid) +#define PTD_RW4(action, a, b, c, d ) PTD_RW3(action, a, b, c) ; ekat::action(&d, 1, fid) +#define PTD_RW5(action, a, b, c, d, e ) PTD_RW4(action, a, b, c, d) ; ekat::action(&e, 1, fid) +#define PTD_RW6(action, a, b, c, d, e, f ) PTD_RW5(action, a, b, c, d, e) ; ekat::action(&f, 1, fid) +#define PTD_RW7(action, a, b, c, d, e, f, g ) PTD_RW6(action, a, b, c, d, e, f) ; ekat::action(&g, 1, fid) +#define PTD_RW8(action, a, b, c, d, e, f, g, h ) PTD_RW7(action, a, b, c, d, e, f, g) ; ekat::action(&h, 1, fid) +#define PTD_RW9(action, a, b, c, d, e, f, g, h, i ) PTD_RW8(action, a, b, c, d, e, f, g, h) ; ekat::action(&i, 1, fid) +#define PTD_RW10(action, a, b, c, d, e, f, g, h, i, j ) PTD_RW9(action, a, b, c, d, e, f, g, h, i) ; ekat::action(&j, 1, fid) +#define PTD_RW11(action, a, b, c, d, e, f, g, h, i, j, k ) PTD_RW10(action, a, b, c, d, e, f, g, h, i, j) ; ekat::action(&k, 1, fid) +#define PTD_RW12(action, a, b, c, d, e, f, g, h, i, j, k, l ) PTD_RW11(action, a, b, c, d, e, f, g, h, i, j, k) ; ekat::action(&l, 1, fid) +#define PTD_RW13(action, a, b, c, d, e, f, g, h, i, j, k, l, m ) PTD_RW12(action, a, b, c, d, e, f, g, h, i, j, k, l) ; ekat::action(&m, 1, fid) +#define PTD_RW14(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n ) PTD_RW13(action, a, b, c, d, e, f, g, h, i, j, k, l, m) ; ekat::action(&n, 1, fid) +#define PTD_RW15(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o ) PTD_RW14(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n) ; ekat::action(&o, 1, fid) +#define PTD_RW16(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p ) PTD_RW15(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) ; ekat::action(&p, 1, fid) +#define PTD_RW17(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q ) PTD_RW16(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) ; ekat::action(&q, 1, fid) +#define PTD_RW18(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r ) PTD_RW17(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) ; ekat::action(&r, 1, fid) +#define PTD_RW19(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s ) PTD_RW18(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) ; ekat::action(&s, 1, fid) +#define PTD_RW20(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t ) PTD_RW19(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) ; ekat::action(&t, 1, fid) #define PTD_ASSIGN_OP(name, num_scalars, ...) \ name& operator=(const name& rhs) { PTD_ASS##num_scalars(__VA_ARGS__); assignment_impl(rhs); return *this; } #define PTD_RW_SCALARS(num_scalars, ...) \ - void read_scalars(const ekat::FILEPtr& fid) { PTD_RW##num_scalars(read, __VA_ARGS__); } \ + void read_scalars(const ekat::FILEPtr& fid) { EKAT_REQUIRE_MSG(fid, "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); PTD_RW##num_scalars(read, __VA_ARGS__); } \ void write_scalars(const ekat::FILEPtr& fid) const { PTD_RW##num_scalars(write, __VA_ARGS__); } #define PTD_RW() \ From cda50baf714be1751516257015dc5fe7f8d60c5d Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 12:49:57 -0600 Subject: [PATCH 209/366] p3_unit_tests and macro cleanup --- .../eamxx/src/physics/p3/p3_functions_f90.hpp | 97 +++++++++++--- .../src/physics/p3/tests/p3_unit_tests.cpp | 126 ++++++++++++++---- .../src/physics/share/physics_test_data.hpp | 102 ++++++++------ 3 files changed, 234 insertions(+), 91 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 20207460c0a..461736bfc64 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -93,6 +93,8 @@ struct LookupIceData // Outputs Int dumi, dumjj, dumii, dumzz; Real dum1, dum4, dum5, dum6; + + PTD_RW_SCALARS_ONLY(8, dumi, dumjj, dumii, dumzz, dum1, dum4, dum5, dum6); }; /////////////////////////////////////////////////////////////////////////////// @@ -105,6 +107,8 @@ struct LookupIceDataB // Outputs Int dumj; Real dum3; + + PTD_RW_SCALARS_ONLY(2, dumj, dum3); }; /////////////////////////////////////////////////////////////////////////////// @@ -117,6 +121,8 @@ struct AccessLookupTableData // Outputs Real proc; + + PTD_RW_SCALARS_ONLY(1, proc); }; /////////////////////////////////////////////////////////////////////////////// @@ -130,6 +136,8 @@ struct AccessLookupTableCollData // Outputs Real proc; + + PTD_RW_SCALARS_ONLY(1, proc); }; /////////////////////////////////////////////////////////////////////////////// @@ -147,6 +155,11 @@ struct BackToCellAverageData // This populates all fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(31, qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend, nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, nr_selfcollect_tend, nr_evap_tend, ncautr, qcnuc, + nc_nuceat_tend, qi2qv_sublim_tend, nr_ice_shed_tend, qc2qi_hetero_freeze_tend, qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, ni2nr_melt_tend, + nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend, ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, + qc2qi_berg_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -158,6 +171,8 @@ struct CloudWaterConservationData //output Real qc2qr_autoconv_tend, qc2qr_accret_tend, qc2qi_collect_tend, qc2qi_hetero_freeze_tend, qc2qr_ice_shed_tend, qc2qi_berg_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend; + + PTD_RW_SCALARS_ONLY(8, qc2qr_autoconv_tend, qc2qr_accret_tend, qc2qi_collect_tend, qc2qi_hetero_freeze_tend, qc2qr_ice_shed_tend, qc2qi_berg_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend); }; struct RainWaterConservationData @@ -167,6 +182,8 @@ struct RainWaterConservationData //output Real qr2qv_evap_tend, qr2qi_collect_tend, qr2qi_immers_freeze_tend; + + PTD_RW_SCALARS_ONLY(3, qr2qv_evap_tend, qr2qi_collect_tend, qr2qi_immers_freeze_tend); }; struct IceWaterConservationData @@ -176,6 +193,8 @@ struct IceWaterConservationData //output Real qi2qv_sublim_tend, qi2qr_melt_tend; + + PTD_RW_SCALARS_ONLY(2, qi2qv_sublim_tend, qi2qr_melt_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -187,6 +206,8 @@ struct CalcRimeDensityData // output Real vtrmi1, rho_qm_cloud; + + PTD_RW_SCALARS_ONLY(2, vtrmi1, rho_qm_cloud); }; /////////////////////////////////////////////////////////////////////////////// @@ -198,6 +219,8 @@ struct CldliqImmersionFreezingData // output Real qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend; + + PTD_RW_SCALARS_ONLY(2, qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -209,6 +232,8 @@ struct RainImmersionFreezingData // output Real qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend; + + PTD_RW_SCALARS_ONLY(2, qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -220,6 +245,8 @@ struct DropletSelfCollectionData // output Real nc_selfcollect_tend; + + PTD_RW_SCALARS_ONLY(1, nc_selfcollect_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -231,6 +258,8 @@ struct CloudRainAccretionData // output Real qc2qr_accret_tend, nc_accret_tend; + + PTD_RW_SCALARS_ONLY(2, qc2qr_accret_tend, nc_accret_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -238,15 +267,12 @@ struct CloudRainAccretionData struct CloudWaterAutoconversionData { // inputs - Real rho; - Real qc_incld; - Real nc_incld; - Real inv_qc_relvar; + Real rho, qc_incld, nc_incld, inv_qc_relvar; // output - Real qc2qr_autoconv_tend; - Real nc2nr_autoconv_tend; - Real ncautr; + Real qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr; + + PTD_RW_SCALARS_ONLY(3, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr); }; /////////////////////////////////////////////////////////////////////////////// @@ -258,6 +284,8 @@ struct RainSelfCollectionData //output Real nr_selfcollect_tend; + + PTD_RW_SCALARS_ONLY(1, nr_selfcollect_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -268,6 +296,8 @@ struct ImposeMaxTotalNiData{ //input Real max_total_ni, inv_rho_local; + + PTD_RW_SCALARS_ONLY(2, ni_local, inv_rho_local); }; /////////////////////////////////////////////////////////////////////////////// @@ -279,6 +309,8 @@ struct IceMeltingData // output Real qi2qr_melt_tend,ni2nr_melt_tend; + + PTD_RW_SCALARS_ONLY(2, qi2qr_melt_tend, ni2nr_melt_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -299,6 +331,8 @@ struct GetCloudDsd2Data // Outputs Real nc_out, mu_c, nu, lamc, cdist, cdist1; + + PTD_RW_SCALARS_ONLY(6, nc_out, mu_c, nu, lamc, cdist, cdist1) }; ////////////////////////////////////////////////////////////////////////// @@ -310,6 +344,8 @@ struct GetRainDsd2Data // Outputs Real nr_out, lamr, mu_r, cdistr, logn0r; + + PTD_RW_SCALARS_ONLY(5, nr_out, lamr, mu_r, cdistr, logn0r); }; /////////////////////////////////////////////////////////////////////////////// @@ -440,6 +476,8 @@ struct CalcBulkRhoRimeData // Outputs Real rho_rime; + + PTD_RW_SCALARS_ONLY(3, qi_rim, bi_rim, rho_rime); }; /////////////////////////////////////////////////////////////////////////////// @@ -475,18 +513,7 @@ struct ComputeRainFallVelocityData // Outputs Real mu_r, lamr, V_qr, V_nr; - PTD_RW_SCALARS(5, nr_incld, mu_r, lamr, V_qr, V_nr); - - void read(const ekat::FILEPtr& fid) - { - read_scalars(fid); - } - - void write(const ekat::FILEPtr& fid) const - { - write_scalars(fid); - } - + PTD_RW_SCALARS_ONLY(5, nr_incld, mu_r, lamr, V_qr, V_nr); }; /////////////////////////////////////////////////////////////////////////////// @@ -498,6 +525,8 @@ struct GetTimeSpacePhysVarsData //Outs Real mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii; + + PTD_RW_SCALARS_ONLY(9, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii); }; /////////////////////////////////////////////////////////////////////////////// @@ -512,6 +541,8 @@ struct P3UpdatePrognosticIceData // In/outs Real th_atm, qv, qi, ni, qm, bm, qc, nc, qr, nr; + + PTD_RW_SCALARS_ONLY(10, th_atm, qv, qi, ni, qm, bm, qc, nc, qr, nr); }; /////////////////////////////////////////////////////////////////////////////// @@ -524,6 +555,8 @@ struct EvapRainData //Outs Real qr2qv_evap_tend, nr_evap_tend; + + PTD_RW_SCALARS_ONLY(2, qr2qv_evap_tend, nr_evap_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -539,6 +572,8 @@ struct P3UpdatePrognosticLiqData // In/outs Real th_atm, qv, qc, nc, qr, nr; + + PTD_RW_SCALARS_ONLY(6, th_atm, qv, qc, nc, qr, nr); }; /////////////////////////////////////////////////////////////////////////////// @@ -554,6 +589,7 @@ struct IceDepositionSublimationData // This populates all input fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + PTD_RW_SCALARS_ONLY(4, qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend); }; struct IceCldliqCollectionData @@ -565,6 +601,7 @@ struct IceCldliqCollectionData // Outputs Real qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc; + PTD_RW_SCALARS_ONLY(4, qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc); }; struct IceRainCollectionData @@ -576,6 +613,7 @@ struct IceRainCollectionData // Outputs Real qr2qi_collect_tend, nr_collect_tend; + PTD_RW_SCALARS_ONLY(2, qr2qi_collect_tend, nr_collect_tend); }; struct IceSelfCollectionData @@ -587,6 +625,7 @@ struct IceSelfCollectionData // Outputs Real ni_selfcollect_tend; + PTD_RW_SCALARS_ONLY(1, ni_selfcollect_tend); }; struct IceRelaxationData @@ -596,6 +635,8 @@ struct IceRelaxationData // Outputs Real epsi, epsi_tot; + + PTD_RW_SCALARS_ONLY(2, epsi, epsi_tot); }; struct CalcLiqRelaxationData @@ -608,6 +649,8 @@ struct CalcLiqRelaxationData // This populates all input fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(2, epsr, epsc); }; struct IceNucleationData @@ -619,6 +662,8 @@ struct IceNucleationData // Outputs Real qv2qi_nucleat_tend, ni_nucleat_tend; + + PTD_RW_SCALARS_ONLY(2, qv2qi_nucleat_tend, ni_nucleat_tend); }; struct IceWetGrowthData @@ -631,6 +676,8 @@ struct IceWetGrowthData bool log_wetgrowth; Real qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend; + + PTD_RW_SCALARS_ONLY(6, log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend); }; struct LatentHeatData : public PhysicsTestData @@ -675,6 +722,8 @@ struct IncloudMixingData // Outputs Real qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld; + + PTD_RW_SCALARS_ONLY(8, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld); }; /////////////////////////////////////////////////////////////////////////////// @@ -796,6 +845,8 @@ struct IceSupersatConservationData { Real qidep, qinuc; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(2, qidep, qinuc); }; struct NcConservationData { @@ -806,6 +857,8 @@ struct NcConservationData { Real nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(4, nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend); }; struct NrConservationData { @@ -816,6 +869,8 @@ struct NrConservationData { Real nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(4, nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend); }; struct NiConservationData { @@ -826,6 +881,8 @@ struct NiConservationData { Real ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(3, ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend); }; struct PreventLiqSupersaturationData { @@ -837,6 +894,8 @@ struct PreventLiqSupersaturationData { // This populates all fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(2, qi2qv_sublim_tend, qr2qv_evap_tend); }; // Glue functions to call fortran from from C++ with the Data struct diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index 2a33cc70db1..9bf22760279 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -222,9 +222,13 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: std::copy(&cwdc[0], &cwdc[0] + max_pack_size, cwdc_host.data()); Kokkos::deep_copy(cwdc_device, cwdc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - cloud_water_conservation(cwdc[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/cloud_water_conservation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + cwdc[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -263,7 +267,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: Kokkos::deep_copy(cwdc_host, cwdc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cwdc[s].qc == cwdc_host(s).qc); REQUIRE(cwdc[s].qc2qr_autoconv_tend == cwdc_host(s).qc2qr_autoconv_tend); @@ -275,6 +279,12 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: REQUIRE(cwdc[s].qv2qi_vapdep_tend == cwdc_host(s).qv2qi_vapdep_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cwdc_host(s).write(fid); + } + } } void ice_water_conservation_unit_bfb_tests() @@ -312,9 +322,13 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: std::copy(&iwdc[0], &iwdc[0] + max_pack_size, iwdc_host.data()); Kokkos::deep_copy(iwdc_device, iwdc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_water_conservation(iwdc[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_water_conservation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + iwdc[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -356,7 +370,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: Kokkos::deep_copy(iwdc_host, iwdc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(iwdc[s].qi == iwdc_host(s).qi); REQUIRE(iwdc[s].qv2qi_vapdep_tend == iwdc_host(s).qv2qi_vapdep_tend ); @@ -370,6 +384,12 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: REQUIRE(iwdc[s].qi2qr_melt_tend == iwdc_host(s).qi2qr_melt_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + iwdc_host(s).write(fid); + } + } } void rain_water_conservation_unit_bfb_tests() { @@ -407,9 +427,13 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: std::copy(&rwdc[0], &rwdc[0] + max_pack_size, rwdc_host.data()); Kokkos::deep_copy(rwdc_device, rwdc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - rain_water_conservation(rwdc[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/rain_water_conservation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + rwdc[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -448,7 +472,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: Kokkos::deep_copy(rwdc_host, rwdc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(rwdc[s].qr == rwdc_host(s).qr); REQUIRE(rwdc[s].qc2qr_autoconv_tend == rwdc_host(s).qc2qr_autoconv_tend); @@ -460,6 +484,12 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: REQUIRE(rwdc[s].qr2qi_immers_freeze_tend == rwdc_host(s).qr2qi_immers_freeze_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + rwdc_host(s).write(fid); + } + } } void run_bfb() { @@ -591,9 +621,13 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitT std::copy(&pupidc[0], &pupidc[0] + max_pack_size, pupidc_host.data()); Kokkos::deep_copy(pupidc_device, pupidc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - update_prognostic_ice(pupidc[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/update_prognostic_ice.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + pupidc[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -679,7 +713,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitT Kokkos::deep_copy(pupidc_host, pupidc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(pupidc[s].th_atm == pupidc_host(s).th_atm); REQUIRE(pupidc[s].qc == pupidc_host(s).qc); @@ -693,6 +727,12 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitT REQUIRE(pupidc[s].bm == pupidc_host(s).bm ); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + pupidc_host(s).write(fid); + } + } } void run_bfb() { @@ -737,9 +777,13 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::U std::copy(>spvd[0], >spvd[0] + max_pack_size, gtspvd_host.data()); Kokkos::deep_copy(gtspvd_device, gtspvd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - get_time_space_phys_variables(gtspvd[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/get_time_space_phys_variables.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + gtspvd[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -794,7 +838,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::U Kokkos::deep_copy(gtspvd_host, gtspvd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(gtspvd[s].mu == gtspvd_host(s).mu); REQUIRE(gtspvd[s].dv == gtspvd_host(s).dv); @@ -807,6 +851,12 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::U REQUIRE(gtspvd[s].eii == gtspvd_host(s).eii); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + gtspvd_host(s).write(fid); + } + } } void run_bfb() { @@ -896,9 +946,13 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitT std::copy(&pupldc[0], &pupldc[0] + max_pack_size, pupldc_host.data()); Kokkos::deep_copy(pupldc_device, pupldc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - update_prognostic_liquid(pupldc[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/update_prognostic_liquid.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + pupldc[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -971,7 +1025,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitT Kokkos::deep_copy(pupldc_host, pupldc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(pupldc[s].th_atm == pupldc_host(s).th_atm); REQUIRE(pupldc[s].qv == pupldc_host(s).qv); @@ -981,6 +1035,12 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitT REQUIRE(pupldc[s].nr == pupldc_host(s).nr); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + pupldc_host(s).write(fid); + } + } } void run_bfb() { @@ -1026,9 +1086,13 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi : public UnitWrap: std::copy(&dc[0], &dc[0] + max_pack_size, dc_host.data()); Kokkos::deep_copy(dc_device, dc_host); - //Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - impose_max_total_ni(dc[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/impose_max_total_ni.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + dc[i].read(fid); + } } //Run function from a kernal and copy results back to the host @@ -1054,12 +1118,18 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi : public UnitWrap: Kokkos::deep_copy(dc_host, dc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(dc[s].ni_local == dc_host(s).ni_local); REQUIRE(dc[s].inv_rho_local == dc_host(s).inv_rho_local); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + dc_host(s).write(fid); + } + } } void run_bfb() { diff --git a/components/eamxx/src/physics/share/physics_test_data.hpp b/components/eamxx/src/physics/share/physics_test_data.hpp index 153ec7b9b28..ddd7d77fa86 100644 --- a/components/eamxx/src/physics/share/physics_test_data.hpp +++ b/components/eamxx/src/physics/share/physics_test_data.hpp @@ -69,50 +69,60 @@ struct SHOCGridData : public PhysicsTestData { #define PTD_DATA_COPY_CTOR(name, num_args) \ name(const name& rhs) : name(PTD_ONES(num_args)) { *this = rhs; } -#define PTD_ASS0( ) ((void) (0)) -#define PTD_ASS1(a ) PTD_ASS0() ; a = rhs.a -#define PTD_ASS2(a, b ) PTD_ASS1(a) ; b = rhs.b -#define PTD_ASS3(a, b, c ) PTD_ASS2(a, b) ; c = rhs.c -#define PTD_ASS4(a, b, c, d ) PTD_ASS3(a, b, c) ; d = rhs.d -#define PTD_ASS5(a, b, c, d, e ) PTD_ASS4(a, b, c, d) ; e = rhs.e -#define PTD_ASS6(a, b, c, d, e, f ) PTD_ASS5(a, b, c, d, e) ; f = rhs.f -#define PTD_ASS7(a, b, c, d, e, f, g ) PTD_ASS6(a, b, c, d, e, f) ; g = rhs.g -#define PTD_ASS8(a, b, c, d, e, f, g, h ) PTD_ASS7(a, b, c, d, e, f, g) ; h = rhs.h -#define PTD_ASS9(a, b, c, d, e, f, g, h, i ) PTD_ASS8(a, b, c, d, e, f, g, h) ; i = rhs.i -#define PTD_ASS10(a, b, c, d, e, f, g, h, i, j ) PTD_ASS9(a, b, c, d, e, f, g, h, i) ; j = rhs.j -#define PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k ) PTD_ASS10(a, b, c, d, e, f, g, h, i, j) ; k = rhs.k -#define PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l ) PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k) ; l = rhs.l -#define PTD_ASS13(a, b, c, d, e, f, g, h, i, j, k, l, m ) PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l) ; m = rhs.m -#define PTD_ASS14(a, b, c, d, e, f, g, h, i, j, k, l, m, n ) PTD_ASS13(a, b, c, d, e, f, g, h, i, j, k, l, m) ; n = rhs.n -#define PTD_ASS15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o ) PTD_ASS14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) ; o = rhs.o -#define PTD_ASS16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p ) PTD_ASS15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) ; p = rhs.p -#define PTD_ASS17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q ) PTD_ASS16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) ; q = rhs.q -#define PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r ) PTD_ASS17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) ; r = rhs.r -#define PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s ) PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) ; s = rhs.s -#define PTD_ASS20(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t ) PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) ; t = rhs.t - - -#define PTD_RW0(action ) ((void) (0)) -#define PTD_RW1(action, a ) PTD_RW0(action) ; ekat::action(&a, 1, fid) -#define PTD_RW2(action, a, b ) PTD_RW1(action, a) ; ekat::action(&b, 1, fid) -#define PTD_RW3(action, a, b, c ) PTD_RW2(action, a, b) ; ekat::action(&c, 1, fid) -#define PTD_RW4(action, a, b, c, d ) PTD_RW3(action, a, b, c) ; ekat::action(&d, 1, fid) -#define PTD_RW5(action, a, b, c, d, e ) PTD_RW4(action, a, b, c, d) ; ekat::action(&e, 1, fid) -#define PTD_RW6(action, a, b, c, d, e, f ) PTD_RW5(action, a, b, c, d, e) ; ekat::action(&f, 1, fid) -#define PTD_RW7(action, a, b, c, d, e, f, g ) PTD_RW6(action, a, b, c, d, e, f) ; ekat::action(&g, 1, fid) -#define PTD_RW8(action, a, b, c, d, e, f, g, h ) PTD_RW7(action, a, b, c, d, e, f, g) ; ekat::action(&h, 1, fid) -#define PTD_RW9(action, a, b, c, d, e, f, g, h, i ) PTD_RW8(action, a, b, c, d, e, f, g, h) ; ekat::action(&i, 1, fid) -#define PTD_RW10(action, a, b, c, d, e, f, g, h, i, j ) PTD_RW9(action, a, b, c, d, e, f, g, h, i) ; ekat::action(&j, 1, fid) -#define PTD_RW11(action, a, b, c, d, e, f, g, h, i, j, k ) PTD_RW10(action, a, b, c, d, e, f, g, h, i, j) ; ekat::action(&k, 1, fid) -#define PTD_RW12(action, a, b, c, d, e, f, g, h, i, j, k, l ) PTD_RW11(action, a, b, c, d, e, f, g, h, i, j, k) ; ekat::action(&l, 1, fid) -#define PTD_RW13(action, a, b, c, d, e, f, g, h, i, j, k, l, m ) PTD_RW12(action, a, b, c, d, e, f, g, h, i, j, k, l) ; ekat::action(&m, 1, fid) -#define PTD_RW14(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n ) PTD_RW13(action, a, b, c, d, e, f, g, h, i, j, k, l, m) ; ekat::action(&n, 1, fid) -#define PTD_RW15(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o ) PTD_RW14(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n) ; ekat::action(&o, 1, fid) -#define PTD_RW16(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p ) PTD_RW15(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) ; ekat::action(&p, 1, fid) -#define PTD_RW17(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q ) PTD_RW16(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) ; ekat::action(&q, 1, fid) -#define PTD_RW18(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r ) PTD_RW17(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) ; ekat::action(&r, 1, fid) -#define PTD_RW19(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s ) PTD_RW18(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) ; ekat::action(&s, 1, fid) -#define PTD_RW20(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t ) PTD_RW19(action, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) ; ekat::action(&t, 1, fid) +#define PTD_ASS0() ((void) (0)) +#define PTD_ASS1(first) first = rhs.first; PTD_ASS0() +#define PTD_ASS2(first, ...) first = rhs.first; PTD_ASS1(__VA_ARGS__) +#define PTD_ASS3(first, ...) first = rhs.first; PTD_ASS2(__VA_ARGS__) +#define PTD_ASS4(first, ...) first = rhs.first; PTD_ASS3(__VA_ARGS__) +#define PTD_ASS5(first, ...) first = rhs.first; PTD_ASS4(__VA_ARGS__) +#define PTD_ASS6(first, ...) first = rhs.first; PTD_ASS5(__VA_ARGS__) +#define PTD_ASS7(first, ...) first = rhs.first; PTD_ASS6(__VA_ARGS__) +#define PTD_ASS8(first, ...) first = rhs.first; PTD_ASS7(__VA_ARGS__) +#define PTD_ASS9(first, ...) first = rhs.first; PTD_ASS8(__VA_ARGS__) +#define PTD_ASS10(first, ...) first = rhs.first; PTD_ASS9(__VA_ARGS__) +#define PTD_ASS11(first, ...) first = rhs.first; PTD_ASS10(__VA_ARGS__) +#define PTD_ASS12(first, ...) first = rhs.first; PTD_ASS11(__VA_ARGS__) +#define PTD_ASS13(first, ...) first = rhs.first; PTD_ASS12(__VA_ARGS__) +#define PTD_ASS14(first, ...) first = rhs.first; PTD_ASS13(__VA_ARGS__) +#define PTD_ASS15(first, ...) first = rhs.first; PTD_ASS14(__VA_ARGS__) +#define PTD_ASS16(first, ...) first = rhs.first; PTD_ASS15(__VA_ARGS__) +#define PTD_ASS17(first, ...) first = rhs.first; PTD_ASS16(__VA_ARGS__) +#define PTD_ASS18(first, ...) first = rhs.first; PTD_ASS17(__VA_ARGS__) +#define PTD_ASS19(first, ...) first = rhs.first; PTD_ASS18(__VA_ARGS__) +#define PTD_ASS20(first, ...) first = rhs.first; PTD_ASS19(__VA_ARGS__) + +#define PTD_RW0(action) ((void) (0)) +#define PTD_RW1(action, first) ekat::action(&first, 1, fid); PTD_RW0(action) +#define PTD_RW2(action, first, ...) ekat::action(&first, 1, fid); PTD_RW1(action, __VA_ARGS__) +#define PTD_RW3(action, first, ...) ekat::action(&first, 1, fid); PTD_RW2(action, __VA_ARGS__) +#define PTD_RW4(action, first, ...) ekat::action(&first, 1, fid); PTD_RW3(action, __VA_ARGS__) +#define PTD_RW5(action, first, ...) ekat::action(&first, 1, fid); PTD_RW4(action, __VA_ARGS__) +#define PTD_RW6(action, first, ...) ekat::action(&first, 1, fid); PTD_RW5(action, __VA_ARGS__) +#define PTD_RW7(action, first, ...) ekat::action(&first, 1, fid); PTD_RW6(action, __VA_ARGS__) +#define PTD_RW8(action, first, ...) ekat::action(&first, 1, fid); PTD_RW7(action, __VA_ARGS__) +#define PTD_RW9(action, first, ...) ekat::action(&first, 1, fid); PTD_RW8(action, __VA_ARGS__) +#define PTD_RW10(action, first, ...) ekat::action(&first, 1, fid); PTD_RW9(action, __VA_ARGS__) +#define PTD_RW11(action, first, ...) ekat::action(&first, 1, fid); PTD_RW10(action, __VA_ARGS__) +#define PTD_RW12(action, first, ...) ekat::action(&first, 1, fid); PTD_RW11(action, __VA_ARGS__) +#define PTD_RW13(action, first, ...) ekat::action(&first, 1, fid); PTD_RW12(action, __VA_ARGS__) +#define PTD_RW14(action, first, ...) ekat::action(&first, 1, fid); PTD_RW13(action, __VA_ARGS__) +#define PTD_RW15(action, first, ...) ekat::action(&first, 1, fid); PTD_RW14(action, __VA_ARGS__) +#define PTD_RW16(action, first, ...) ekat::action(&first, 1, fid); PTD_RW15(action, __VA_ARGS__) +#define PTD_RW17(action, first, ...) ekat::action(&first, 1, fid); PTD_RW16(action, __VA_ARGS__) +#define PTD_RW18(action, first, ...) ekat::action(&first, 1, fid); PTD_RW17(action, __VA_ARGS__) +#define PTD_RW19(action, first, ...) ekat::action(&first, 1, fid); PTD_RW18(action, __VA_ARGS__) +#define PTD_RW20(action, first, ...) ekat::action(&first, 1, fid); PTD_RW19(action, __VA_ARGS__) +#define PTD_RW21(action, first, ...) ekat::action(&first, 1, fid); PTD_RW20(action, __VA_ARGS__) +#define PTD_RW22(action, first, ...) ekat::action(&first, 1, fid); PTD_RW21(action, __VA_ARGS__) +#define PTD_RW23(action, first, ...) ekat::action(&first, 1, fid); PTD_RW22(action, __VA_ARGS__) +#define PTD_RW24(action, first, ...) ekat::action(&first, 1, fid); PTD_RW23(action, __VA_ARGS__) +#define PTD_RW25(action, first, ...) ekat::action(&first, 1, fid); PTD_RW24(action, __VA_ARGS__) +#define PTD_RW26(action, first, ...) ekat::action(&first, 1, fid); PTD_RW25(action, __VA_ARGS__) +#define PTD_RW27(action, first, ...) ekat::action(&first, 1, fid); PTD_RW26(action, __VA_ARGS__) +#define PTD_RW28(action, first, ...) ekat::action(&first, 1, fid); PTD_RW27(action, __VA_ARGS__) +#define PTD_RW29(action, first, ...) ekat::action(&first, 1, fid); PTD_RW28(action, __VA_ARGS__) +#define PTD_RW30(action, first, ...) ekat::action(&first, 1, fid); PTD_RW29(action, __VA_ARGS__) +#define PTD_RW31(action, first, ...) ekat::action(&first, 1, fid); PTD_RW30(action, __VA_ARGS__) #define PTD_ASSIGN_OP(name, num_scalars, ...) \ name& operator=(const name& rhs) { PTD_ASS##num_scalars(__VA_ARGS__); assignment_impl(rhs); return *this; } @@ -121,6 +131,10 @@ struct SHOCGridData : public PhysicsTestData { void read_scalars(const ekat::FILEPtr& fid) { EKAT_REQUIRE_MSG(fid, "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); PTD_RW##num_scalars(read, __VA_ARGS__); } \ void write_scalars(const ekat::FILEPtr& fid) const { PTD_RW##num_scalars(write, __VA_ARGS__); } +#define PTD_RW_SCALARS_ONLY(num_scalars, ...) \ + void read(const ekat::FILEPtr& fid) { EKAT_REQUIRE_MSG(fid, "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); PTD_RW##num_scalars(read, __VA_ARGS__); } \ + void write(const ekat::FILEPtr& fid) const { PTD_RW##num_scalars(write, __VA_ARGS__); } + #define PTD_RW() \ void read(const ekat::FILEPtr& fid) { read_scalars(fid); PhysicsTestData::read(fid); } \ void write(const ekat::FILEPtr& fid) const { write_scalars(fid); PhysicsTestData::write(fid); } From c3560ef4e9c167c759533b2070bd0d948215ca56 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 13:33:14 -0600 Subject: [PATCH 210/366] p3_cloud_water_autoconversion_test --- .../p3/tests/p3_autoconversion_unit_tests.cpp | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp index e4ff1fd4366..39a8d143544 100644 --- a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp @@ -59,12 +59,16 @@ void cloud_water_autoconversion_unit_bfb_tests() { std::copy(&cwadc[0], &cwadc[0] + max_pack_size, cwadc_host.data()); Kokkos::deep_copy(cwadc_device, cwadc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - cloud_water_autoconversion(cwadc[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/cloud_water_autoconversion.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + cwadc[i].read(fid); + } } - // Run the lookup from a kernel and copy results back to host + // Run the lookup from a kernel and copy results back to host Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { const Int offset = i * Spack::n; @@ -101,7 +105,7 @@ void cloud_water_autoconversion_unit_bfb_tests() { Kokkos::deep_copy(cwadc_host, cwadc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cwadc[s].rho == cwadc_host(s).rho); REQUIRE(cwadc[s].qc_incld == cwadc_host(s).qc_incld); @@ -112,6 +116,12 @@ void cloud_water_autoconversion_unit_bfb_tests() { REQUIRE(cwadc[s].ncautr == cwadc_host(s).ncautr); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cwadc_host(s).write(fid); + } + } } void run_bfb() { From cd284196a1f5ce30548d2f6be080ab33de0beda2 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 13:40:25 -0600 Subject: [PATCH 211/366] p3_back_to_cell_average was not following the standard pattern --- .../p3_back_to_cell_average_unit_tests.cpp | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index f73bd3d993a..796eb6912c0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -32,32 +32,34 @@ void run_bfb() // Generate n test structs, each populated with random data (values within // [0,1]) by the default constructor. - BackToCellAverageData back_to_cell_average_data[Spack::n]; - for (Int i = 0; i < Spack::n; ++i) { - back_to_cell_average_data[i].randomize(engine); + BackToCellAverageData back_to_cell_average_data[max_pack_size]; + for (auto& item : back_to_cell_average_data) { + item.randomize(engine); } // Sync to device. - view_1d device_data("back_to_cell_average", Spack::n); + view_1d device_data("back_to_cell_average", max_pack_size); const auto host_data = Kokkos::create_mirror_view(device_data); - std::copy(&back_to_cell_average_data[0], &back_to_cell_average_data[0] + Spack::n, + std::copy(&back_to_cell_average_data[0], &back_to_cell_average_data[0] + max_pack_size, host_data.data()); Kokkos::deep_copy(device_data, host_data); // Run the Fortran subroutine. - for (Int i = 0; i < Spack::n; ++i) { + for (Int i = 0; i < max_pack_size; ++i) { back_to_cell_average(back_to_cell_average_data[i]); } // Run the lookup from a kernel and copy results back to host - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { + const Int offset = i * Spack::n; + // Init pack inputs Spack cld_frac_l, cld_frac_r, cld_frac_i, qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend, nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, nr_selfcollect_tend, nr_evap_tend, ncautr, qi2qv_sublim_tend, nr_ice_shed_tend, qc2qi_hetero_freeze_tend, qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, ni2nr_melt_tend, nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend, ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, qc2qi_berg_tend; - for (Int s = 0; s < Spack::n; ++s) { + for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { cld_frac_l[s] = device_data[s].cld_frac_l; cld_frac_r[s] = device_data[s].cld_frac_r; cld_frac_i[s] = device_data[s].cld_frac_i; @@ -99,7 +101,7 @@ void run_bfb() nr_collect_tend, ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, qc2qi_berg_tend); // Copy results back into views - for (Int s = 0; s < Spack::n; ++s) { + for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { device_data(s).qc2qr_accret_tend = qc2qr_accret_tend[s]; device_data(s).qr2qv_evap_tend = qr2qv_evap_tend[s]; device_data(s).qc2qr_autoconv_tend = qc2qr_autoconv_tend[s]; @@ -137,7 +139,7 @@ void run_bfb() // Validate results. if (SCREAM_BFB_TESTING) { - for (Int s = 0; s < Spack::n; ++s) { + for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(back_to_cell_average_data[s].qc2qr_accret_tend == host_data[s].qc2qr_accret_tend); REQUIRE(back_to_cell_average_data[s].qr2qv_evap_tend == host_data[s].qr2qv_evap_tend); REQUIRE(back_to_cell_average_data[s].qc2qr_autoconv_tend == host_data[s].qc2qr_autoconv_tend); From a28704098c25b0a5bd629efa01de0f947b5055a7 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 13:44:40 -0600 Subject: [PATCH 212/366] p3_back_to_cell_average --- .../p3_back_to_cell_average_unit_tests.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index 796eb6912c0..58066bb6eac 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -28,7 +28,7 @@ void run_phys() void run_bfb() { - auto engine = setup_random_test(); + auto engine = setup_random_test(314552); // Generate n test structs, each populated with random data (values within // [0,1]) by the default constructor. @@ -48,6 +48,14 @@ void run_bfb() for (Int i = 0; i < max_pack_size; ++i) { back_to_cell_average(back_to_cell_average_data[i]); } + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/back_to_cell_average.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + back_to_cell_average_data[i].read(fid); + } + } // Run the lookup from a kernel and copy results back to host Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { @@ -138,7 +146,7 @@ void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(back_to_cell_average_data[s].qc2qr_accret_tend == host_data[s].qc2qr_accret_tend); REQUIRE(back_to_cell_average_data[s].qr2qv_evap_tend == host_data[s].qr2qv_evap_tend); @@ -171,6 +179,12 @@ void run_bfb() REQUIRE(back_to_cell_average_data[s].qc2qi_berg_tend == host_data[s].qc2qi_berg_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(fid); + } + } } }; From eae74a877d48627db88989810e1eb54fe7dfb2f5 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 13:53:47 -0600 Subject: [PATCH 213/366] Two more easy ones --- ...lc_liq_relaxation_timescale_unit_tests.cpp | 20 ++++++++++++++----- .../tests/p3_calc_rime_density_unit_tests.cpp | 18 +++++++++++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index 69715267176..c0da609087c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -29,7 +29,7 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: void run_bfb() { - auto engine = setup_random_test(); + auto engine = setup_random_test(12354); // Read in tables view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; @@ -54,9 +54,13 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: self[i].f2r = C::f2r; } - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - calc_liq_relaxation_timescale(self[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/calc_liq_relaxation_timescale.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(fid); + } } // Sync to device @@ -97,12 +101,18 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].epsr == self_host(s).epsr); REQUIRE(self[s].epsc == self_host(s).epsc); } } + else { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(fid); + } + } } }; diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp index f858797fa00..1fee033460b 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp @@ -87,9 +87,13 @@ void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - calc_rime_density(calc_rime_density_data[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/calc_rime_density.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + calc_rime_density_data[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -126,12 +130,18 @@ void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(calc_rime_density_data[s].vtrmi1 == host_data[s].vtrmi1); REQUIRE(calc_rime_density_data[s].rho_qm_cloud == host_data[s].rho_qm_cloud); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(fid); + } + } } }; From ac5851d3150fa5fdfd535b44eb49c968912c1c97 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 14:06:31 -0600 Subject: [PATCH 214/366] Four more --- .../p3/tests/p3_check_values_unit_tests.cpp | 21 +------ .../p3_cldliq_imm_freezing_unit_tests.cpp | 18 ++++-- .../p3/tests/p3_cloud_rain_acc_unit_tests.cpp | 18 ++++-- .../p3/tests/p3_cloud_sed_unit_tests.cpp | 56 +++++++++++-------- 4 files changed, 64 insertions(+), 49 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp index 8b9ecf3904f..15d76d346f4 100644 --- a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp @@ -24,9 +24,10 @@ struct UnitWrap::UnitTest::TestCheckValues : public UnitWrap::UnitTest::Ba void run_check_values_bfb() { + // This is not really a bfb test since no results are being checked. auto engine = setup_random_test(); - CheckValuesData cvd_fortran[] = { + CheckValuesData cvd_cxx[] = { // kts_, kte_, timestepcount_, source_ind_, force_abort_ CheckValuesData(1, 72, 2, 100, false), CheckValuesData(1, 72, 3, 100, false), @@ -34,26 +35,10 @@ void run_check_values_bfb() CheckValuesData(1, 72, 5, 100, false), }; - static constexpr Int num_runs = sizeof(cvd_fortran) / sizeof(CheckValuesData); - - for (auto& d : cvd_fortran) { + for (auto& d : cvd_cxx) { d.randomize(engine, { {d.qv, {-4.056E-01, 1.153E+00}}, {d.temp, {1.000E+02, 5.000E+02}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - CheckValuesData cvd_cxx[num_runs] = { - CheckValuesData(cvd_fortran[0]), - CheckValuesData(cvd_fortran[1]), - CheckValuesData(cvd_fortran[2]), - CheckValuesData(cvd_fortran[3]), - }; - - // Get data from fortran - for (auto& d : cvd_fortran) { - check_values(d); - } - // Get data from cxx for (auto& d : cvd_cxx) { check_values_f(d.qv, d.temp, d.kts, d.kte, d.timestepcount, d.force_abort, d.source_ind, d.col_loc); diff --git a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp index 901650d8695..9c152443405 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp @@ -70,9 +70,13 @@ void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - cldliq_immersion_freezing(cldliq_imm_freezing_data[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/cldliq_imm_freezing.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + cldliq_imm_freezing_data[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -109,12 +113,18 @@ void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cldliq_imm_freezing_data[s].qc2qi_hetero_freeze_tend == host_data[s].qc2qi_hetero_freeze_tend); REQUIRE(cldliq_imm_freezing_data[s].nc2ni_immers_freeze_tend == host_data[s].nc2ni_immers_freeze_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(fid); + } + } } }; diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp index 2cfe22e5a26..276187f8ec0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp @@ -73,9 +73,13 @@ void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - cloud_rain_accretion(cloud_rain_acc_data[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/cloud_rain_accretion.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + cloud_rain_acc_data[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -112,12 +116,18 @@ void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cloud_rain_acc_data[s].qc2qr_accret_tend == host_data[s].qc2qr_accret_tend); REQUIRE(cloud_rain_acc_data[s].nc_accret_tend == host_data[s].nc_accret_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(fid); + } + } } }; diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index b11400c98f4..ace2523fc2d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -28,9 +28,9 @@ void run_phys() void run_bfb() { - auto engine = setup_random_test(); + auto engine = setup_random_test(23512); - CloudSedData csds_fortran[] = { + CloudSedData csds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, do_predict_nc, precip_liq_surf, CloudSedData(1, 72, 27, 72, -1, 1.800E+03, 5.556E-04, false, 0.0), CloudSedData(1, 72, 72, 27, 1, 1.800E+03, 5.556E-04, false, 0.0), @@ -39,26 +39,30 @@ void run_bfb() CloudSedData(1, 72, 27, 27, -1, 1.800E+03, 5.556E-04, true, 0.0), }; - static constexpr Int num_runs = sizeof(csds_fortran) / sizeof(CloudSedData); + static constexpr Int num_runs = sizeof(csds_baseline) / sizeof(CloudSedData); // Set up random input data - for (auto& d : csds_fortran) { + for (auto& d : csds_baseline) { d.randomize(engine, { {d.qc_incld, {C::QSMALL/2, C::QSMALL*2}} }); } // Create copies of data for use by cxx. Needs to happen before fortran calls so that // inout data is in original state CloudSedData csds_cxx[num_runs] = { - CloudSedData(csds_fortran[0]), - CloudSedData(csds_fortran[1]), - CloudSedData(csds_fortran[2]), - CloudSedData(csds_fortran[3]), - CloudSedData(csds_fortran[4]), + CloudSedData(csds_baseline[0]), + CloudSedData(csds_baseline[1]), + CloudSedData(csds_baseline[2]), + CloudSedData(csds_baseline[3]), + CloudSedData(csds_baseline[4]), }; - // Get data from fortran - for (auto& d : csds_fortran) { - cloud_sedimentation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/cloud_sedimentation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (auto& d : csds_baseline) { + d.read(fid); + } } // Get data from cxx @@ -69,21 +73,27 @@ void run_bfb() d.qc, d.nc, d.nc_incld, d.mu_c, d.lamc, &d.precip_liq_surf, d.qc_tend, d.nc_tend); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(csds_fortran[i].kbot, csds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(csds_fortran[i].kbot, csds_fortran[i].ktop); // 0-based indx + Int start = std::min(csds_baseline[i].kbot, csds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(csds_baseline[i].kbot, csds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(csds_fortran[i].qc[k] == csds_cxx[i].qc[k]); - REQUIRE(csds_fortran[i].nc[k] == csds_cxx[i].nc[k]); - REQUIRE(csds_fortran[i].nc_incld[k] == csds_cxx[i].nc_incld[k]); - REQUIRE(csds_fortran[i].mu_c[k] == csds_cxx[i].mu_c[k]); - REQUIRE(csds_fortran[i].lamc[k] == csds_cxx[i].lamc[k]); - REQUIRE(csds_fortran[i].qc_tend[k] == csds_cxx[i].qc_tend[k]); - REQUIRE(csds_fortran[i].nc_tend[k] == csds_cxx[i].nc_tend[k]); + REQUIRE(csds_baseline[i].qc[k] == csds_cxx[i].qc[k]); + REQUIRE(csds_baseline[i].nc[k] == csds_cxx[i].nc[k]); + REQUIRE(csds_baseline[i].nc_incld[k] == csds_cxx[i].nc_incld[k]); + REQUIRE(csds_baseline[i].mu_c[k] == csds_cxx[i].mu_c[k]); + REQUIRE(csds_baseline[i].lamc[k] == csds_cxx[i].lamc[k]); + REQUIRE(csds_baseline[i].qc_tend[k] == csds_cxx[i].qc_tend[k]); + REQUIRE(csds_baseline[i].nc_tend[k] == csds_cxx[i].nc_tend[k]); } - REQUIRE(csds_fortran[i].precip_liq_surf == csds_cxx[i].precip_liq_surf); + REQUIRE(csds_baseline[i].precip_liq_surf == csds_cxx[i].precip_liq_surf); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + csds_cxx[i].write(fid); } } } From 3a64d91391f40b64729b549f438872e15f7225d7 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 14:25:14 -0600 Subject: [PATCH 215/366] get_latent_heat is not being used on the cxx side --- .../eamxx/src/physics/p3/p3_functions_f90.cpp | 16 ---- .../eamxx/src/physics/p3/p3_functions_f90.hpp | 16 ---- .../tests/p3_droplet_self_coll_unit_tests.cpp | 19 ++++- .../physics/p3/tests/p3_dsd2_unit_tests.cpp | 36 +++++++-- .../p3/tests/p3_evaporate_rain_unit_tests.cpp | 18 ++++- .../tests/p3_get_latent_heat_unit_tests.cpp | 79 ------------------- 6 files changed, 57 insertions(+), 127 deletions(-) delete mode 100644 components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index 7ce3f8aaa32..6b027935801 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -176,8 +176,6 @@ void ice_cldliq_wet_growth_c(Real rho, Real temp, Real pres, Real rhofaci, Real Real qi_incld, Real ni_incld, Real qr_incld, bool* log_wetgrowth, Real* qr2qi_collect_tend, Real* qc2qi_collect_tend, Real* qc_growth_rate, Real* nr_ice_shed_tend, Real* qc2qr_ice_shed_tend); -void get_latent_heat_c(Int its, Int ite, Int kts, Int kte, Real* s, Real* v, Real* f); - Real subgrid_variance_scaling_c(Real relvar, Real expon); void check_values_c(Real* qv, Real* temp, Int kts, Int kte, Int timestepcount, @@ -347,20 +345,6 @@ void cldliq_immersion_freezing(CldliqImmersionFreezingData& d) &d.qc2qi_hetero_freeze_tend, &d.nc2ni_immers_freeze_tend); } -LatentHeatData::LatentHeatData(Int kts_, Int kte_, Int its_, Int ite_) : - PhysicsTestData( { {(ite_ - its_) + 1, (kte_ - kts_) + 1} }, - { {&v, &s, &f} }), - its(its_), ite(ite_), kts(kts_), kte(kte_) -{} - -void get_latent_heat(LatentHeatData& d) -{ - p3_init(); - d.transpose(); - get_latent_heat_c(d.its, d.ite, d.kts, d.kte, d.v, d.s, d.f); - d.transpose(); -} - void droplet_self_collection(DropletSelfCollectionData& d) { p3_init(); diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 461736bfc64..40b92fe47b6 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -680,21 +680,6 @@ struct IceWetGrowthData PTD_RW_SCALARS_ONLY(6, log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend); }; -struct LatentHeatData : public PhysicsTestData -{ - static constexpr size_t NUM_ARRAYS = 3; - - // Inputs - Int its, ite, kts, kte; - - // Outputs - Real* v, *s, *f; - - LatentHeatData(Int its_, Int ite_, Int kts_, Int kte_); - - PTD_STD_DEF(LatentHeatData, 4, its, ite, kts, kte); -}; - struct CheckValuesData : public PhysicsTestData { static constexpr size_t NUM_ARRAYS = 2; @@ -940,7 +925,6 @@ void ice_relaxation_timescale(IceRelaxationData& d); void calc_liq_relaxation_timescale(CalcLiqRelaxationData& d); void ice_nucleation(IceNucleationData& d); void ice_cldliq_wet_growth(IceWetGrowthData& d); -void get_latent_heat(LatentHeatData& d); void check_values(CheckValuesData& d); void calculate_incloud_mixingratios(IncloudMixingData& d); void p3_main_part1(P3MainPart1Data& d); diff --git a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp index fccf41db627..2a96b1b3bff 100644 --- a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp @@ -72,9 +72,13 @@ void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - droplet_self_collection(droplet_self_coll_data[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/droplet_self_collection.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + droplet_self_coll_data[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -107,11 +111,18 @@ void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(droplet_self_coll_data[s].nc_selfcollect_tend == host_data[s].nc_selfcollect_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(fid); + } + } + } }; diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 43c5a7de8dd..6563335acb0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -60,9 +60,13 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { std::copy(&gcdd[0], &gcdd[0] + max_pack_size, gcdd_host.data()); Kokkos::deep_copy(gcdd_device, gcdd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - get_cloud_dsd2(gcdd[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/get_cloud_dsd2.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + gcdd[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -95,7 +99,7 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { Kokkos::deep_copy(gcdd_host, gcdd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(gcdd[s].nc_out == gcdd_host(s).nc_out); REQUIRE(gcdd[s].mu_c == gcdd_host(s).mu_c); @@ -105,6 +109,12 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { REQUIRE(gcdd[s].cdist1 == gcdd_host(s).cdist1); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + gcdd_host(s).write(fid); + } + } } void run_cloud_phys() @@ -144,9 +154,13 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { std::copy(&grdd[0], &grdd[0] + max_pack_size, grdd_host.data()); Kokkos::deep_copy(grdd_device, grdd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - get_rain_dsd2(grdd[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/get_rain_dsd2.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + grdd[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -179,7 +193,7 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { Kokkos::deep_copy(grdd_host, grdd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(grdd[s].nr_out == grdd_host(s).nr_out); REQUIRE(grdd[s].mu_r == grdd_host(s).mu_r); @@ -188,6 +202,12 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { REQUIRE(grdd[s].logn0r == grdd_host(s).logn0r); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + grdd_host(s).write(fid); + } + } } void run_rain_phys() diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index 9df3ecdb0c8..c5626e0387c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -173,9 +173,13 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest: std::copy(&espd[0], &espd[0] + max_pack_size, espd_host.data()); Kokkos::deep_copy(espd_device, espd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - evaporate_rain(espd[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/evaporate_rain.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + espd[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -241,12 +245,18 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest: Kokkos::deep_copy(espd_host, espd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(espd[s].qr2qv_evap_tend == espd_host(s).qr2qv_evap_tend); REQUIRE(espd[s].nr_evap_tend == espd_host(s).nr_evap_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + espd_host(s).write(fid); + } + } } // end run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp deleted file mode 100644 index dfe6f3ca6a4..00000000000 --- a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp +++ /dev/null @@ -1,79 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" - -#include "p3_unit_tests_common.hpp" - -#include -#include -#include -#include -#include // std::setprecision - -namespace scream { -namespace p3 { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestLatentHeat : public UnitWrap::UnitTest::Base { - - void run_latent_heat_bfb() - { - constexpr Scalar latvap = C::LatVap; - constexpr Scalar latice = C::LatIce; - - LatentHeatData latent_fortran[] = { - // its, ite, kts, kte - LatentHeatData(1, 7, 1, 10), - }; - - static constexpr Int num_runs = sizeof(latent_fortran) / sizeof(LatentHeatData); - - LatentHeatData latent_cxx[num_runs] = { - LatentHeatData(latent_fortran[0]), - LatentHeatData(latent_fortran[1]), - LatentHeatData(latent_fortran[2]), - LatentHeatData(latent_fortran[3]), - }; - - for (Int i = 0; i < num_runs; ++i) { - LatentHeatData& h = latent_fortran[i]; - get_latent_heat(h); - - if (SCREAM_BFB_TESTING) { - for (Int j = 0; j < h.total(h.v); ++j) { - REQUIRE(h.v[j] == latvap); - REQUIRE(h.s[j] == (latvap+latice)); - REQUIRE(h.f[j] == latice); - } - } - } - } - - void run_latent_heat_phys() - { - // TODO - } -}; - -} -} -} - -namespace { - -TEST_CASE("p3_latent_heat", "[p3_functions]") -{ - using T = scream::p3::unit_test::UnitWrap::UnitTest::TestLatentHeat; - - T t; - t.run_latent_heat_phys(); - t.run_latent_heat_bfb(); -} - -} From dd7f4539890adea185fb6ef44eb1f51f00e5137c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 14:55:31 -0600 Subject: [PATCH 216/366] More --- .../p3_ice_cldliq_wet_growth_unit_tests.cpp | 18 ++- .../p3/tests/p3_ice_collection_unit_tests.cpp | 54 +++++-- .../p3_ice_deposition_sublimation_tests.cpp | 77 +++++----- .../p3/tests/p3_ice_melting_unit_tests.cpp | 19 ++- .../p3/tests/p3_ice_nucleation_unit_tests.cpp | 21 ++- ...p3_ice_relaxation_timescale_unit_tests.cpp | 20 ++- .../p3/tests/p3_ice_sed_unit_tests.cpp | 145 +++++++++++------- 7 files changed, 231 insertions(+), 123 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index e2a61b37481..5060d9d2fdc 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -57,9 +57,13 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest std::copy(&self[0], &self[0] + max_pack_size, self_host.data()); Kokkos::deep_copy(self_device, self_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_cldliq_wet_growth(self[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_cldliq_wet_growth.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -114,7 +118,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(static_cast(self[s].log_wetgrowth) == static_cast(self_host(s).log_wetgrowth)); @@ -125,6 +129,12 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest REQUIRE(self[s].qc2qr_ice_shed_tend == self_host(s).qc2qr_ice_shed_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(fid); + } + } } void run_ice_cldliq_wet_growth_phys() diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index 9c628c0641c..6f34526e9b0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -63,9 +63,13 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: std::copy(&cldliq[0], &cldliq[0] + max_pack_size, cldliq_host.data()); Kokkos::deep_copy(cldliq_device, cldliq_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_cldliq_collection(cldliq[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_cldliq_collection.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + cldliq[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -109,7 +113,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: Kokkos::deep_copy(cldliq_host, cldliq_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cldliq[s].qc2qi_collect_tend == cldliq_host(s).qc2qi_collect_tend); REQUIRE(cldliq[s].nc_collect_tend == cldliq_host(s).nc_collect_tend); @@ -117,6 +121,12 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: REQUIRE(cldliq[s].ncshdc == cldliq_host(s).ncshdc); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cldliq_host(s).write(fid); + } + } } void run_ice_cldliq_phys() @@ -157,9 +167,13 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: std::copy(&rain[0], &rain[0] + max_pack_size, rain_host.data()); Kokkos::deep_copy(rain_device, rain_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_rain_collection(rain[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_rain_collection.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + rain[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -198,12 +212,18 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: Kokkos::deep_copy(rain_host, rain_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(rain[s].qr2qi_collect_tend == rain_host(s).qr2qi_collect_tend); REQUIRE(rain[s].nr_collect_tend == rain_host(s).nr_collect_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + rain_host(s).write(fid); + } + } } void run_ice_rain_phys() @@ -244,9 +264,13 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: std::copy(&self[0], &self[0] + max_pack_size, self_host.data()); Kokkos::deep_copy(self_device, self_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_self_collection(self[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_self_collection.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -276,11 +300,17 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].ni_selfcollect_tend == self_host(s).ni_selfcollect_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(fid); + } + } } void run_ice_self_phys() diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp index f6afd915dc6..b359ad661e9 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp @@ -49,7 +49,7 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un void run_bfb() { - IceDepositionSublimationData f90_data[max_pack_size] = { + IceDepositionSublimationData baseline_data[max_pack_size] = { {1.0000E-04,4.5010E+05,2.8750E+02,1.1279E-02,1.1279E-02,0.0000E+00,3.3648E+00,5.0000E-03,1.666667e-02}, {5.1000E-03,4.5370E+05,2.8542E+02,9.9759E-03,9.9759E-03,0.0000E+00,3.1223E+00,5.0000E-03,1.666667e-02}, {5.1000E-03,4.5742E+05,2.8334E+02,8.8076E-03,8.8076E-03,0.0000E+00,2.9014E+00,5.0000E-03,1.666667e-02}, @@ -68,11 +68,9 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un {5.0000E-08,5.4479E+05,2.4793E+02,7.5430E-04,5.8895E-04,4.6769E-04,1.1661E+00,1.5278E-04,1.666667e-02}, }; - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IceDepositionSublimationData); - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - //for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + //for (auto& d : baseline_data) { // d.randomize(); //} @@ -80,12 +78,16 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - ice_deposition_sublimation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_deposition_sublimation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(fid); + } } // Get data from cxx. Run ice_deposition_sublimation from a kernel and copy results back to host @@ -110,7 +112,6 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un // Init outputs Spack ni_sublim_tend(0), qi2qv_sublim_tend(0), qc2qi_berg_tend(0), qv2qi_vapdep_tend(0); - Functions::ice_deposition_sublimation(qi_incld, ni_incld, T_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt, qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend); // Copy spacks back into cxx_device view @@ -120,35 +121,41 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un cxx_device(vs).qc2qi_berg_tend = qc2qi_berg_tend[s]; cxx_device(vs).qv2qi_vapdep_tend = qv2qi_vapdep_tend[s]; } - }); Kokkos::deep_copy(cxx_host, cxx_device); - for (Int i = 0; i < num_runs; ++i) { - // Verify BFB results - IceDepositionSublimationData& d_f90 = f90_data[i]; - IceDepositionSublimationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.qv2qi_vapdep_tend == d_cxx.qv2qi_vapdep_tend); - REQUIRE(d_f90.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); - REQUIRE(d_f90.ni_sublim_tend == d_cxx.ni_sublim_tend); - REQUIRE(d_f90.qc2qi_berg_tend == d_cxx.qc2qi_berg_tend); - - //MAKE SURE OUTPUT IS WITHIN EXPECTED BOUNDS: - REQUIRE(d_cxx.qv2qi_vapdep_tend >=0); - REQUIRE(d_cxx.qi2qv_sublim_tend >=0); - REQUIRE(d_cxx.ni_sublim_tend >=0); - REQUIRE(d_cxx.qc2qi_berg_tend >=0); - - //vapdep should only occur when qv>qv_sat_i - REQUIRE( (d_cxx.qv2qi_vapdep_tend==0 || d_cxx.qv + d_cxx.qv2qi_vapdep_tend*d_cxx.inv_dt >= d_cxx.qv_sat_i) ); - //sublim should only occur when qvfrz, berg and vapdep should be 0: - REQUIRE( (d_cxx.T_atmm_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + // Verify BFB results + IceDepositionSublimationData& d_f90 = baseline_data[i]; + IceDepositionSublimationData& d_cxx = cxx_host[i]; + REQUIRE(d_f90.qv2qi_vapdep_tend == d_cxx.qv2qi_vapdep_tend); + REQUIRE(d_f90.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); + REQUIRE(d_f90.ni_sublim_tend == d_cxx.ni_sublim_tend); + REQUIRE(d_f90.qc2qi_berg_tend == d_cxx.qc2qi_berg_tend); + + //MAKE SURE OUTPUT IS WITHIN EXPECTED BOUNDS: + REQUIRE(d_cxx.qv2qi_vapdep_tend >=0); + REQUIRE(d_cxx.qi2qv_sublim_tend >=0); + REQUIRE(d_cxx.ni_sublim_tend >=0); + REQUIRE(d_cxx.qc2qi_berg_tend >=0); + + //vapdep should only occur when qv>qv_sat_i + REQUIRE( (d_cxx.qv2qi_vapdep_tend==0 || d_cxx.qv + d_cxx.qv2qi_vapdep_tend*d_cxx.inv_dt >= d_cxx.qv_sat_i) ); + //sublim should only occur when qvfrz, berg and vapdep should be 0: + REQUIRE( (d_cxx.T_atmm_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(fid); + } } } // run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index 5e29579a910..86c42707c1e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -57,9 +57,13 @@ void ice_melting_bfb() { std::copy(&IceMelt[0], &IceMelt[0] + max_pack_size, IceMelt_host.data()); Kokkos::deep_copy(IceMelt_device, IceMelt_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_melting(IceMelt[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_melting.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + IceMelt[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -99,12 +103,19 @@ void ice_melting_bfb() { Kokkos::deep_copy(IceMelt_host, IceMelt_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(IceMelt[s].qi2qr_melt_tend == IceMelt_host(s).qi2qr_melt_tend); REQUIRE(IceMelt[s].ni2nr_melt_tend == IceMelt_host(s).ni2nr_melt_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + IceMelt_host(s).write(fid); + } + } + } }; // UnitWrap diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp index 51ab5031c45..154adbabb15 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp @@ -54,10 +54,15 @@ struct UnitWrap::UnitTest::TestIceNucleation : public UnitWrap::UnitTest:: {2.702E+02, 1.069E+00, 0.323E+03, 2.221E+01, 9.952E-01, inv_dt, do_predict_nc, do_prescribed_CCN } }; - // Run the fortran code - for (Int i = 0; i < max_pack_size; ++i) { - ice_nucleation(self[i]); - } + std::string root_name = "ice_nucleation"; + std::string file_name = root_name + (do_predict_nc ? "1" : "0") + (do_prescribed_CCN ? "1" : "0"); + std::string baseline_name = this->m_baseline_path + "/" + file_name + ".dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(fid); + } + } // Sync to device KTH::view_1d self_host("self_host", max_pack_size); @@ -94,12 +99,18 @@ struct UnitWrap::UnitTest::TestIceNucleation : public UnitWrap::UnitTest:: Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].qv2qi_nucleat_tend == self_host(s).qv2qi_nucleat_tend); REQUIRE(self[s].ni_nucleat_tend == self_host(s).ni_nucleat_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(fid); + } + } } //end for do_predict_nc } //end for do_prescribed_CCN } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp index dfb668bc81d..7015b7a94f8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp @@ -49,10 +49,14 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale : public UnitWrap::Unit {1.352E+01, 3.210E+03, 1.069E+00, 0.123E+00, 3.456E+00, 1.221E-02, 9.952E-07, 6.596E-05, 4.532E-01, 1.734E+04} }; - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_relaxation_timescale(self[i]); - } + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_relaxation_timescale.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(fid); + } + } // Sync to device KTH::view_1d self_host("self_host", max_pack_size); @@ -93,12 +97,18 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale : public UnitWrap::Unit Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].epsi == self_host(s).epsi); REQUIRE(self[s].epsi_tot == self_host(s).epsi_tot); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(fid); + } + } } void run_ice_relaxation_timescale_phys() diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index 86ff76e9169..3afcc6af92b 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -48,7 +48,7 @@ void run_bfb_calc_bulk_rhime() constexpr Scalar qsmall = C::QSMALL; // Load some lookup inputs, need at least one per pack value - CalcBulkRhoRimeData cbrr_fortran[max_pack_size] = { + CalcBulkRhoRimeData cbrr_baseline[max_pack_size] = { // qi_tot, qi_rim, bi_rim {9.999978E-08, 9.999978E-03, 1.111108E-10}, {0.000000E+00, 8.571428E-05, 1.000000E-02}, @@ -75,13 +75,15 @@ void run_bfb_calc_bulk_rhime() // inout data is in original state view_1d cbrr_device("cbrr", max_pack_size); const auto cbrr_host = Kokkos::create_mirror_view(cbrr_device); - std::copy(&cbrr_fortran[0], &cbrr_fortran[0] + max_pack_size, cbrr_host.data()); + std::copy(&cbrr_baseline[0], &cbrr_baseline[0] + max_pack_size, cbrr_host.data()); Kokkos::deep_copy(cbrr_device, cbrr_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - if (cbrr_fortran[i].qi_tot > qsmall) { - calc_bulk_rho_rime(cbrr_fortran[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/calc_bulk_rho_rime.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + cbrr_baseline[i].read(fid); } } @@ -114,20 +116,26 @@ void run_bfb_calc_bulk_rhime() Kokkos::deep_copy(cbrr_host, cbrr_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { - REQUIRE(cbrr_fortran[s].qi_rim == cbrr_host(s).qi_rim); - REQUIRE(cbrr_fortran[s].bi_rim == cbrr_host(s).bi_rim); - REQUIRE(cbrr_fortran[s].rho_rime == cbrr_host(s).rho_rime); + REQUIRE(cbrr_baseline[s].qi_rim == cbrr_host(s).qi_rim); + REQUIRE(cbrr_baseline[s].bi_rim == cbrr_host(s).bi_rim); + REQUIRE(cbrr_baseline[s].rho_rime == cbrr_host(s).rho_rime); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cbrr_host(s).write(fid); } } } void run_bfb_ice_sed() { - auto engine = setup_random_test(); + auto engine = setup_random_test(124135); - IceSedData isds_fortran[] = { + IceSedData isds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, precip_ice_surf IceSedData(1, 72, 27, 72, -1, 1.800E+03, 5.556E-04, 0.0), IceSedData(1, 72, 72, 27, 1, 1.800E+03, 5.556E-04, 1.0), @@ -135,25 +143,29 @@ void run_bfb_ice_sed() IceSedData(1, 72, 27, 27, 1, 1.800E+03, 5.556E-04, 2.0), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(IceSedData); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(IceSedData); // Set up random input data - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { d.randomize(engine, { {d.qi_incld, {C::QSMALL/2, C::QSMALL*2}} }); } // Create copies of data for use by cxx. Needs to happen before fortran calls so that // inout data is in original state IceSedData isds_cxx[num_runs] = { - IceSedData(isds_fortran[0]), - IceSedData(isds_fortran[1]), - IceSedData(isds_fortran[2]), - IceSedData(isds_fortran[3]), + IceSedData(isds_baseline[0]), + IceSedData(isds_baseline[1]), + IceSedData(isds_baseline[2]), + IceSedData(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - ice_sedimentation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_sedimentation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < num_runs; ++i) { + isds_baseline[i].read(fid); + } } // Get data from cxx @@ -165,24 +177,30 @@ void run_bfb_ice_sed() d.ni_incld, &d.precip_ice_surf, d.qi_tend, d.ni_tend); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); - REQUIRE(isds_fortran[i].qi_tend[k] == isds_cxx[i].qi_tend[k]); - REQUIRE(isds_fortran[i].ni_tend[k] == isds_cxx[i].ni_tend[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); + REQUIRE(isds_baseline[i].qi_tend[k] == isds_cxx[i].qi_tend[k]); + REQUIRE(isds_baseline[i].ni_tend[k] == isds_cxx[i].ni_tend[k]); } - REQUIRE(isds_fortran[i].precip_ice_surf == isds_cxx[i].precip_ice_surf); + REQUIRE(isds_baseline[i].precip_ice_surf == isds_cxx[i].precip_ice_surf); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(fid); } } } @@ -191,9 +209,9 @@ void run_bfb_homogeneous_freezing() { constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = setup_random_test(13543563); - HomogeneousFreezingData hfds_fortran[] = { + HomogeneousFreezingData hfds_baseline[] = { // kts, kte, ktop, kbot, kdir HomogeneousFreezingData(1, 72, 27, 72, -1), HomogeneousFreezingData(1, 72, 72, 27, 1), @@ -201,10 +219,10 @@ void run_bfb_homogeneous_freezing() HomogeneousFreezingData(1, 72, 27, 27, 1), }; - static constexpr Int num_runs = sizeof(hfds_fortran) / sizeof(HomogeneousFreezingData); + static constexpr Int num_runs = sizeof(hfds_baseline) / sizeof(HomogeneousFreezingData); // Set up random input data - for (auto& d : hfds_fortran) { + for (auto& d : hfds_baseline) { const auto qsmall_r = std::make_pair(C::QSMALL/2, C::QSMALL*2); d.randomize(engine, { {d.T_atm, {C::T_homogfrz - 10, C::T_homogfrz + 10}}, {d.qc, qsmall_r}, {d.qr, qsmall_r} }); @@ -218,15 +236,19 @@ void run_bfb_homogeneous_freezing() // Create copies of data for use by cxx. Needs to happen before fortran calls so that // inout data is in original state HomogeneousFreezingData hfds_cxx[num_runs] = { - HomogeneousFreezingData(hfds_fortran[0]), - HomogeneousFreezingData(hfds_fortran[1]), - HomogeneousFreezingData(hfds_fortran[2]), - HomogeneousFreezingData(hfds_fortran[3]), + HomogeneousFreezingData(hfds_baseline[0]), + HomogeneousFreezingData(hfds_baseline[1]), + HomogeneousFreezingData(hfds_baseline[2]), + HomogeneousFreezingData(hfds_baseline[3]), }; - // Get data from fortran - for (auto& d : hfds_fortran) { - homogeneous_freezing(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/homogeneous_freezing.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (auto& d : hfds_baseline) { + d.read(fid); + } } // Get data from cxx @@ -236,24 +258,31 @@ void run_bfb_homogeneous_freezing() d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.th_atm); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(hfds_fortran[i].kbot, hfds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(hfds_fortran[i].kbot, hfds_fortran[i].ktop); // 0-based indx + Int start = std::min(hfds_baseline[i].kbot, hfds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(hfds_baseline[i].kbot, hfds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(hfds_fortran[i].qc[k] == hfds_cxx[i].qc[k]); - REQUIRE(hfds_fortran[i].nc[k] == hfds_cxx[i].nc[k]); - REQUIRE(hfds_fortran[i].qr[k] == hfds_cxx[i].qr[k]); - REQUIRE(hfds_fortran[i].nr[k] == hfds_cxx[i].nr[k]); - REQUIRE(hfds_fortran[i].qi[k] == hfds_cxx[i].qi[k]); - REQUIRE(hfds_fortran[i].ni[k] == hfds_cxx[i].ni[k]); - REQUIRE(hfds_fortran[i].qm[k] == hfds_cxx[i].qm[k]); - REQUIRE(hfds_fortran[i].bm[k] == hfds_cxx[i].bm[k]); - REQUIRE(hfds_fortran[i].th_atm[k] == hfds_cxx[i].th_atm[k]); + REQUIRE(hfds_baseline[i].qc[k] == hfds_cxx[i].qc[k]); + REQUIRE(hfds_baseline[i].nc[k] == hfds_cxx[i].nc[k]); + REQUIRE(hfds_baseline[i].qr[k] == hfds_cxx[i].qr[k]); + REQUIRE(hfds_baseline[i].nr[k] == hfds_cxx[i].nr[k]); + REQUIRE(hfds_baseline[i].qi[k] == hfds_cxx[i].qi[k]); + REQUIRE(hfds_baseline[i].ni[k] == hfds_cxx[i].ni[k]); + REQUIRE(hfds_baseline[i].qm[k] == hfds_cxx[i].qm[k]); + REQUIRE(hfds_baseline[i].bm[k] == hfds_cxx[i].bm[k]); + REQUIRE(hfds_baseline[i].th_atm[k] == hfds_cxx[i].th_atm[k]); } } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + hfds_cxx[i].write(fid); + } + } + } void run_bfb() From 343c6d6d789888a7581ec57a87710a67682302c4 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 1 Nov 2024 16:07:46 -0600 Subject: [PATCH 217/366] More --- .../eamxx/src/physics/p3/p3_functions_f90.cpp | 6 +- .../eamxx/src/physics/p3/p3_functions_f90.hpp | 13 +- .../p3_ice_supersat_conservation_tests.cpp | 33 +- .../p3/tests/p3_ice_tables_unit_tests.cpp | 60 ++- .../p3_incloud_mixingratios_unit_tests.cpp | 18 +- .../physics/p3/tests/p3_main_unit_tests.cpp | 374 ++++++++++-------- 6 files changed, 296 insertions(+), 208 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index 6b027935801..b5e051a4d11 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -701,7 +701,7 @@ void compute_rain_fall_velocity(ComputeRainFallVelocityData& d) P3MainPart1Data::P3MainPart1Data( Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_) : + bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, bool, bool) : PhysicsTestData( { {(kte_ - kts_) + 1} }, { { &pres, &dpres, &dz, &nc_nuceat_tend, &inv_exner, &exner, &inv_cld_frac_l, &inv_cld_frac_i, &inv_cld_frac_r, &latent_heat_vapor, &latent_heat_sublim, &latent_heat_fusion, &nccn_prescribed, &T_atm, &rho, &inv_rho, &qv_sat_l, &qv_sat_i, &qv_supersat_i, &rhofacr, &rhofaci, @@ -730,7 +730,7 @@ void p3_main_part1(P3MainPart1Data& d) P3MainPart2Data::P3MainPart2Data( Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_) : + bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, Real, bool) : PhysicsTestData( { {(kte_ - kts_) + 1} }, { { &pres, &dpres, &dz, &nc_nuceat_tend, &inv_exner, &exner, &inv_cld_frac_l, &inv_cld_frac_i, &inv_cld_frac_r, &ni_activated, &inv_qc_relvar, &cld_frac_i, &cld_frac_l, &cld_frac_r, &qv_prev, &t_prev, &T_atm, &rho, &inv_rho, &qv_sat_l, &qv_sat_i, &qv_supersat_i, &rhofacr, &rhofaci, &acn, @@ -785,7 +785,7 @@ void p3_main_part3(P3MainPart3Data& d) /////////////////////////////////////////////////////////////////////////////// P3MainData::P3MainData( - Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_) : + Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_, Real) : PhysicsTestData( { {(ite_ - its_) + 1, (kte_ - kts_) + 1}, {(ite_ - its_) + 1, (kte_ - kts_) + 2} }, { { &pres, &dz, &nc_nuceat_tend, &nccn_prescribed, &ni_activated, &dpres, &inv_exner, &cld_frac_i, &cld_frac_l, &cld_frac_r, &inv_qc_relvar, &qc, &nc, &qr, &nr, &qi, &qm, &ni, &bm, &qv, &th_atm, &qv_prev, &t_prev, diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 40b92fe47b6..881356f3d9b 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -733,9 +733,9 @@ struct P3MainPart1Data : public PhysicsTestData bool is_nucleat_possible, is_hydromet_present; P3MainPart1Data(Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_); + bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, bool=false, bool=false); - PTD_STD_DEF(P3MainPart1Data, 8, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt); + PTD_STD_DEF(P3MainPart1Data, 10, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, is_nucleat_possible, is_hydromet_present); Int nk() const { return (kte - kts) + 1; } }; @@ -762,10 +762,9 @@ struct P3MainPart2Data : public PhysicsTestData bool is_hydromet_present; P3MainPart2Data(Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN, Real dt_); + bool do_predict_nc_, bool do_prescribed_CCN, Real dt_, Real=0., bool=false); - PTD_DATA_COPY_CTOR(P3MainPart2Data, 8); - PTD_ASSIGN_OP(P3MainPart2Data, 10, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, is_hydromet_present); + PTD_STD_DEF(P3MainPart2Data, 10, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, is_hydromet_present); Int nk() const { return (kte - kts) + 1; } }; @@ -817,9 +816,9 @@ struct P3MainData : public PhysicsTestData *precip_liq_flux, *precip_ice_flux, *precip_liq_surf, *precip_ice_surf; Real elapsed_s; - P3MainData(Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_); + P3MainData(Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_, Real=0.); - PTD_STD_DEF(P3MainData, 8, its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN); + PTD_STD_DEF(P3MainData, 9, its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN, elapsed_s); }; struct IceSupersatConservationData { diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp index 3c8121cdf3c..e71463f1e25 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp @@ -21,15 +21,15 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::Uni constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = setup_random_test(124151); - IceSupersatConservationData f90_data[max_pack_size]; + IceSupersatConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // hold this fixed, it is not packed data + d.dt = baseline_data[0].dt; // hold this fixed, it is not packed data // C++ impl uses constants for latent_heat values. Manually set here // so F90 can match @@ -40,12 +40,16 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::Uni // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - ice_supersat_conservation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ice_supersat_conservation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(fid); + } } // Get data from cxx. Run ice_supersat_conservation from a kernel and copy results back to host @@ -72,20 +76,25 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::Uni cxx_device(vs).qidep = qidep[s]; cxx_device(vs).qinuc = qinuc[s]; } - }); Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - IceSupersatConservationData& d_f90 = f90_data[i]; + IceSupersatConservationData& d_f90 = baseline_data[i]; IceSupersatConservationData& d_cxx = cxx_host[i]; REQUIRE(d_f90.qidep == d_cxx.qidep); REQUIRE(d_f90.qinuc == d_cxx.qinuc); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(fid); + } + } } // run_bfb }; diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp index 0b171b0a7ff..98f61766860 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp @@ -199,14 +199,6 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base {lid[15], lidb[15], access_table_index} }; - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - find_lookuptable_indices_1a(lid[i]); - find_lookuptable_indices_1b(lidb[i]); - access_lookup_table(altd[i]); - access_lookup_table_coll(altcd[i]); - } - // Sync to device KTH::view_1d lid_host("lid_host", max_pack_size); KTH::view_1d lidb_host("lidb_host", max_pack_size); @@ -217,6 +209,18 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base Kokkos::deep_copy(lid_device, lid_host); Kokkos::deep_copy(lidb_device, lidb_host); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/p3_ice_tables_all.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + lid[i].read(fid); + lidb[i].read(fid); + altd[i].read(fid); + altcd[i].read(fid); + } + } + // Run the lookup from a kernel and copy results back to host view_2d int_results("int results", 5, max_pack_size); view_2d real_results("real results", 7, max_pack_size); @@ -270,15 +274,14 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base Kokkos::deep_copy(real_results_mirror, real_results); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for(int s = 0; s < max_pack_size; ++s) { - // +1 for O vs 1-based indexing - REQUIRE(int_results_mirror(0, s)+1 == lid[s].dumi); - REQUIRE(int_results_mirror(1, s)+1 == lid[s].dumjj); - REQUIRE(int_results_mirror(2, s)+1 == lid[s].dumii); - REQUIRE(int_results_mirror(3, s)+1 == lid[s].dumzz); + REQUIRE(int_results_mirror(0, s) == lid[s].dumi); + REQUIRE(int_results_mirror(1, s) == lid[s].dumjj); + REQUIRE(int_results_mirror(2, s) == lid[s].dumii); + REQUIRE(int_results_mirror(3, s) == lid[s].dumzz); - REQUIRE(int_results_mirror(4, s)+1 == lidb[s].dumj); + REQUIRE(int_results_mirror(4, s) == lidb[s].dumj); REQUIRE(real_results_mirror(0, s) == lid[s].dum1); REQUIRE(real_results_mirror(1, s) == lid[s].dum4); @@ -292,6 +295,33 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base REQUIRE(real_results_mirror(6, s) == altcd[s].proc); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + lid[s].dumi = int_results_mirror(0, s); + lid[s].dumjj = int_results_mirror(1, s); + lid[s].dumii = int_results_mirror(2, s); + lid[s].dumzz = int_results_mirror(3, s); + + lidb[s].dumj = int_results_mirror(4, s); + + lid[s].dum1 = real_results_mirror(0, s); + lid[s].dum4 = real_results_mirror(1, s); + lid[s].dum5 = real_results_mirror(2, s); + lid[s].dum6 = real_results_mirror(3, s); + + lidb[s].dum3 = real_results_mirror(4, s); + + altd[s].proc = real_results_mirror(5, s); + + altcd[s].proc = real_results_mirror(6, s); + + lid[s].write(fid); + lidb[s].write(fid); + altd[s].write(fid); + altcd[s].write(fid); + } + } } void run_phys() diff --git a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp index a75bdbd1728..fd768e70839 100644 --- a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp @@ -69,9 +69,13 @@ struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest:: std::copy(&self[0], &self[0] + max_pack_size, self_host.data()); Kokkos::deep_copy(self_device, self_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - calculate_incloud_mixingratios(self[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/calculate_incloud_mixingratios.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -115,7 +119,7 @@ struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest:: Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].qc_incld == self_host(s).qc_incld); REQUIRE(self[s].qr_incld == self_host(s).qr_incld); @@ -127,6 +131,12 @@ struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest:: REQUIRE(self[s].bm_incld == self_host(s).bm_incld); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(fid); + } + } } void run_incloud_mixing_phys() diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index 35089c0e7f9..62a7e19dddb 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -51,7 +51,7 @@ void run_phys() void run_bfb_p3_main_part1() { - auto engine = setup_random_test(); + auto engine = setup_random_test(125125); constexpr Scalar qsmall = C::QSMALL; //PMC wouldn't it make more sense to define qsmall at a higher level since used in part1, part2, and part3? constexpr Scalar T_zerodegc = C::T_zerodegc; @@ -60,7 +60,7 @@ void run_bfb_p3_main_part1() constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - P3MainPart1Data isds_fortran[] = { + P3MainPart1Data isds_baseline[] = { // kts, kte, ktop, kbot, kdir, do_predict_nc, do_prescribed_CCN, dt P3MainPart1Data(1, 72, 1, 72, 1, false, true, 1.800E+03), P3MainPart1Data(1, 72, 1, 72, 1, true, true, 1.800E+03), @@ -68,9 +68,9 @@ void run_bfb_p3_main_part1() P3MainPart1Data(1, 72, 72, 1, -1, true, false, 1.800E+03), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainPart1Data); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainPart1Data); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { const auto qsmall_r = std::make_pair(0, qsmall*2); //PMC this range seems inappropriately small d.randomize(engine, { {d.T_atm, {T_zerodegc - 10, T_zerodegc + 10}}, @@ -86,18 +86,22 @@ void run_bfb_p3_main_part1() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainPart1Data isds_cxx[num_runs] = { - P3MainPart1Data(isds_fortran[0]), - P3MainPart1Data(isds_fortran[1]), - P3MainPart1Data(isds_fortran[2]), - P3MainPart1Data(isds_fortran[3]), + P3MainPart1Data(isds_baseline[0]), + P3MainPart1Data(isds_baseline[1]), + P3MainPart1Data(isds_baseline[2]), + P3MainPart1Data(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main_part1(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/p3_main_part1.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (auto& d : isds_baseline) { + d.read(fid); + } } // Get data from cxx @@ -111,48 +115,54 @@ void run_bfb_p3_main_part1() &d.is_nucleat_possible, &d.is_hydromet_present); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].T_atm[k] == isds_cxx[i].T_atm[k]); - REQUIRE(isds_fortran[i].rho[k] == isds_cxx[i].rho[k]); - REQUIRE(isds_fortran[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); - REQUIRE(isds_fortran[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); - REQUIRE(isds_fortran[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); - REQUIRE(isds_fortran[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); - REQUIRE(isds_fortran[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); - REQUIRE(isds_fortran[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); - REQUIRE(isds_fortran[i].acn[k] == isds_cxx[i].acn[k]); - REQUIRE(isds_fortran[i].qv[k] == isds_cxx[i].qv[k]); - REQUIRE(isds_fortran[i].th_atm[k] == isds_cxx[i].th_atm[k]); - REQUIRE(isds_fortran[i].qc[k] == isds_cxx[i].qc[k]); - REQUIRE(isds_fortran[i].nc[k] == isds_cxx[i].nc[k]); - REQUIRE(isds_fortran[i].qr[k] == isds_cxx[i].qr[k]); - REQUIRE(isds_fortran[i].nr[k] == isds_cxx[i].nr[k]); - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); - REQUIRE(isds_fortran[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); - REQUIRE(isds_fortran[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); - REQUIRE(isds_fortran[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); - REQUIRE(isds_fortran[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); - REQUIRE(isds_fortran[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); - REQUIRE(isds_fortran[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); - REQUIRE(isds_fortran[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); + REQUIRE(isds_baseline[i].T_atm[k] == isds_cxx[i].T_atm[k]); + REQUIRE(isds_baseline[i].rho[k] == isds_cxx[i].rho[k]); + REQUIRE(isds_baseline[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); + REQUIRE(isds_baseline[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); + REQUIRE(isds_baseline[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); + REQUIRE(isds_baseline[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); + REQUIRE(isds_baseline[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); + REQUIRE(isds_baseline[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); + REQUIRE(isds_baseline[i].acn[k] == isds_cxx[i].acn[k]); + REQUIRE(isds_baseline[i].qv[k] == isds_cxx[i].qv[k]); + REQUIRE(isds_baseline[i].th_atm[k] == isds_cxx[i].th_atm[k]); + REQUIRE(isds_baseline[i].qc[k] == isds_cxx[i].qc[k]); + REQUIRE(isds_baseline[i].nc[k] == isds_cxx[i].nc[k]); + REQUIRE(isds_baseline[i].qr[k] == isds_cxx[i].qr[k]); + REQUIRE(isds_baseline[i].nr[k] == isds_cxx[i].nr[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); + REQUIRE(isds_baseline[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); + REQUIRE(isds_baseline[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); + REQUIRE(isds_baseline[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); + REQUIRE(isds_baseline[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); + REQUIRE(isds_baseline[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); + REQUIRE(isds_baseline[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); + REQUIRE(isds_baseline[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); } - REQUIRE( isds_fortran[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); - REQUIRE( isds_fortran[i].is_nucleat_possible == isds_cxx[i].is_nucleat_possible ); + REQUIRE( isds_baseline[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); + REQUIRE( isds_baseline[i].is_nucleat_possible == isds_cxx[i].is_nucleat_possible ); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(fid); } } } void run_bfb_p3_main_part2() { - auto engine = setup_random_test(); + auto engine = setup_random_test(263267); constexpr Scalar qsmall = C::QSMALL; constexpr Scalar T_zerodegc = C::T_zerodegc; @@ -161,7 +171,7 @@ void run_bfb_p3_main_part2() constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - P3MainPart2Data isds_fortran[] = { + P3MainPart2Data isds_baseline[] = { // kts, kte, ktop, kbot, kdir, do_predict_nc, do_prescribed_CCN, dt P3MainPart2Data(1, 72, 1, 72, 1, false, true, 1.800E+03), P3MainPart2Data(1, 72, 1, 72, 1, true, true, 1.800E+03), @@ -169,9 +179,9 @@ void run_bfb_p3_main_part2() P3MainPart2Data(1, 72, 72, 1, -1, true, false, 1.800E+03), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainPart2Data); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainPart2Data); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { const auto qsmall_r = std::make_pair(0, qsmall*2); d.randomize(engine, { {d.T_atm, {T_zerodegc - 10, T_zerodegc + 10}}, @@ -188,18 +198,22 @@ void run_bfb_p3_main_part2() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainPart2Data isds_cxx[num_runs] = { - P3MainPart2Data(isds_fortran[0]), - P3MainPart2Data(isds_fortran[1]), - P3MainPart2Data(isds_fortran[2]), - P3MainPart2Data(isds_fortran[3]), + P3MainPart2Data(isds_baseline[0]), + P3MainPart2Data(isds_baseline[1]), + P3MainPart2Data(isds_baseline[2]), + P3MainPart2Data(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main_part2(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/p3_main_part2.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (auto& d : isds_baseline) { + d.read(fid); + } } // Get data from cxx @@ -215,61 +229,67 @@ void run_bfb_p3_main_part2() d.prctot, &d.is_hydromet_present); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].T_atm[k] == isds_cxx[i].T_atm[k]); - REQUIRE(isds_fortran[i].rho[k] == isds_cxx[i].rho[k]); - REQUIRE(isds_fortran[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); - REQUIRE(isds_fortran[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); - REQUIRE(isds_fortran[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); - REQUIRE(isds_fortran[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); - REQUIRE(isds_fortran[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); - REQUIRE(isds_fortran[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); - REQUIRE(isds_fortran[i].acn[k] == isds_cxx[i].acn[k]); - REQUIRE(isds_fortran[i].qv[k] == isds_cxx[i].qv[k]); - REQUIRE(isds_fortran[i].th_atm[k] == isds_cxx[i].th_atm[k]); - REQUIRE(isds_fortran[i].qc[k] == isds_cxx[i].qc[k]); - REQUIRE(isds_fortran[i].nc[k] == isds_cxx[i].nc[k]); - REQUIRE(isds_fortran[i].qr[k] == isds_cxx[i].qr[k]); - REQUIRE(isds_fortran[i].nr[k] == isds_cxx[i].nr[k]); - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].latent_heat_vapor[k] == latvap); - REQUIRE(isds_fortran[i].latent_heat_sublim[k] == (latvap+latice)); - REQUIRE(isds_fortran[i].latent_heat_fusion[k] == latice); - REQUIRE(isds_fortran[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); - REQUIRE(isds_fortran[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); - REQUIRE(isds_fortran[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); - REQUIRE(isds_fortran[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); - REQUIRE(isds_fortran[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); - REQUIRE(isds_fortran[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); - REQUIRE(isds_fortran[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); - REQUIRE(isds_fortran[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); - REQUIRE(isds_fortran[i].mu_c[k] == isds_cxx[i].mu_c[k]); - REQUIRE(isds_fortran[i].nu[k] == isds_cxx[i].nu[k]); - REQUIRE(isds_fortran[i].lamc[k] == isds_cxx[i].lamc[k]); - REQUIRE(isds_fortran[i].cdist[k] == isds_cxx[i].cdist[k]); - REQUIRE(isds_fortran[i].cdist1[k] == isds_cxx[i].cdist1[k]); - REQUIRE(isds_fortran[i].cdistr[k] == isds_cxx[i].cdistr[k]); - REQUIRE(isds_fortran[i].mu_r[k] == isds_cxx[i].mu_r[k]); - REQUIRE(isds_fortran[i].lamr[k] == isds_cxx[i].lamr[k]); - REQUIRE(isds_fortran[i].logn0r[k] == isds_cxx[i].logn0r[k]); - REQUIRE(isds_fortran[i].qv2qi_depos_tend[k] == isds_cxx[i].qv2qi_depos_tend[k]); - REQUIRE(isds_fortran[i].precip_total_tend[k] == isds_cxx[i].precip_total_tend[k]); - REQUIRE(isds_fortran[i].nevapr[k] == isds_cxx[i].nevapr[k]); - REQUIRE(isds_fortran[i].qr_evap_tend[k] == isds_cxx[i].qr_evap_tend[k]); - REQUIRE(isds_fortran[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); - REQUIRE(isds_fortran[i].vap_ice_exchange[k] == isds_cxx[i].vap_ice_exchange[k]); - REQUIRE(isds_fortran[i].liq_ice_exchange[k] == isds_cxx[i].liq_ice_exchange[k]); - REQUIRE(isds_fortran[i].pratot[k] == isds_cxx[i].pratot[k]); - REQUIRE(isds_fortran[i].prctot[k] == isds_cxx[i].prctot[k]); + REQUIRE(isds_baseline[i].T_atm[k] == isds_cxx[i].T_atm[k]); + REQUIRE(isds_baseline[i].rho[k] == isds_cxx[i].rho[k]); + REQUIRE(isds_baseline[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); + REQUIRE(isds_baseline[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); + REQUIRE(isds_baseline[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); + REQUIRE(isds_baseline[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); + REQUIRE(isds_baseline[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); + REQUIRE(isds_baseline[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); + REQUIRE(isds_baseline[i].acn[k] == isds_cxx[i].acn[k]); + REQUIRE(isds_baseline[i].qv[k] == isds_cxx[i].qv[k]); + REQUIRE(isds_baseline[i].th_atm[k] == isds_cxx[i].th_atm[k]); + REQUIRE(isds_baseline[i].qc[k] == isds_cxx[i].qc[k]); + REQUIRE(isds_baseline[i].nc[k] == isds_cxx[i].nc[k]); + REQUIRE(isds_baseline[i].qr[k] == isds_cxx[i].qr[k]); + REQUIRE(isds_baseline[i].nr[k] == isds_cxx[i].nr[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].latent_heat_vapor[k] == latvap); + REQUIRE(isds_baseline[i].latent_heat_sublim[k] == (latvap+latice)); + REQUIRE(isds_baseline[i].latent_heat_fusion[k] == latice); + REQUIRE(isds_baseline[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); + REQUIRE(isds_baseline[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); + REQUIRE(isds_baseline[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); + REQUIRE(isds_baseline[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); + REQUIRE(isds_baseline[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); + REQUIRE(isds_baseline[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); + REQUIRE(isds_baseline[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); + REQUIRE(isds_baseline[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); + REQUIRE(isds_baseline[i].mu_c[k] == isds_cxx[i].mu_c[k]); + REQUIRE(isds_baseline[i].nu[k] == isds_cxx[i].nu[k]); + REQUIRE(isds_baseline[i].lamc[k] == isds_cxx[i].lamc[k]); + REQUIRE(isds_baseline[i].cdist[k] == isds_cxx[i].cdist[k]); + REQUIRE(isds_baseline[i].cdist1[k] == isds_cxx[i].cdist1[k]); + REQUIRE(isds_baseline[i].cdistr[k] == isds_cxx[i].cdistr[k]); + REQUIRE(isds_baseline[i].mu_r[k] == isds_cxx[i].mu_r[k]); + REQUIRE(isds_baseline[i].lamr[k] == isds_cxx[i].lamr[k]); + REQUIRE(isds_baseline[i].logn0r[k] == isds_cxx[i].logn0r[k]); + REQUIRE(isds_baseline[i].qv2qi_depos_tend[k] == isds_cxx[i].qv2qi_depos_tend[k]); + REQUIRE(isds_baseline[i].precip_total_tend[k] == isds_cxx[i].precip_total_tend[k]); + REQUIRE(isds_baseline[i].nevapr[k] == isds_cxx[i].nevapr[k]); + REQUIRE(isds_baseline[i].qr_evap_tend[k] == isds_cxx[i].qr_evap_tend[k]); + REQUIRE(isds_baseline[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); + REQUIRE(isds_baseline[i].vap_ice_exchange[k] == isds_cxx[i].vap_ice_exchange[k]); + REQUIRE(isds_baseline[i].liq_ice_exchange[k] == isds_cxx[i].liq_ice_exchange[k]); + REQUIRE(isds_baseline[i].pratot[k] == isds_cxx[i].pratot[k]); + REQUIRE(isds_baseline[i].prctot[k] == isds_cxx[i].prctot[k]); } - REQUIRE( isds_fortran[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); + REQUIRE( isds_baseline[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(fid); } } } @@ -279,11 +299,11 @@ void run_bfb_p3_main_part3() constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = setup_random_test(3734734); constexpr Scalar qsmall = C::QSMALL; - P3MainPart3Data isds_fortran[] = { + P3MainPart3Data isds_baseline[] = { // kts, kte, ktop, kbot, kdir P3MainPart3Data(1, 72, 1, 72, 1), P3MainPart3Data(1, 72, 1, 72, 1), @@ -291,9 +311,9 @@ void run_bfb_p3_main_part3() P3MainPart3Data(1, 72, 72, 1, -1), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainPart3Data); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainPart3Data); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { const auto qsmall_r = std::make_pair(0, qsmall*2); d.randomize(engine, { {d.qc, qsmall_r}, {d.qr, qsmall_r}, {d.qi, qsmall_r} }); @@ -305,18 +325,22 @@ void run_bfb_p3_main_part3() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainPart3Data isds_cxx[num_runs] = { - P3MainPart3Data(isds_fortran[0]), - P3MainPart3Data(isds_fortran[1]), - P3MainPart3Data(isds_fortran[2]), - P3MainPart3Data(isds_fortran[3]), + P3MainPart3Data(isds_baseline[0]), + P3MainPart3Data(isds_baseline[1]), + P3MainPart3Data(isds_baseline[2]), + P3MainPart3Data(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main_part3(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/p3_main_part3.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (auto& d : isds_baseline) { + d.read(fid); + } } // Get data from cxx @@ -329,59 +353,65 @@ void run_bfb_p3_main_part3() d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc, d.diag_eff_radius_qr); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].rho[k] == isds_cxx[i].rho[k]); - REQUIRE(isds_fortran[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); - REQUIRE(isds_fortran[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); - REQUIRE(isds_fortran[i].qv[k] == isds_cxx[i].qv[k]); - REQUIRE(isds_fortran[i].th_atm[k] == isds_cxx[i].th_atm[k]); - REQUIRE(isds_fortran[i].qc[k] == isds_cxx[i].qc[k]); - REQUIRE(isds_fortran[i].nc[k] == isds_cxx[i].nc[k]); - REQUIRE(isds_fortran[i].qr[k] == isds_cxx[i].qr[k]); - REQUIRE(isds_fortran[i].nr[k] == isds_cxx[i].nr[k]); - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].latent_heat_vapor[k] == latvap); - REQUIRE(isds_fortran[i].latent_heat_sublim[k] == latvap+latice); - REQUIRE(isds_fortran[i].mu_c[k] == isds_cxx[i].mu_c[k]); - REQUIRE(isds_fortran[i].nu[k] == isds_cxx[i].nu[k]); - REQUIRE(isds_fortran[i].lamc[k] == isds_cxx[i].lamc[k]); - REQUIRE(isds_fortran[i].mu_r[k] == isds_cxx[i].mu_r[k]); - REQUIRE(isds_fortran[i].lamr[k] == isds_cxx[i].lamr[k]); - REQUIRE(isds_fortran[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); - REQUIRE(isds_fortran[i].ze_rain[k] == isds_cxx[i].ze_rain[k]); - REQUIRE(isds_fortran[i].ze_ice[k] == isds_cxx[i].ze_ice[k]); - REQUIRE(isds_fortran[i].diag_vm_qi[k] == isds_cxx[i].diag_vm_qi[k]); - REQUIRE(isds_fortran[i].diag_eff_radius_qi[k] == isds_cxx[i].diag_eff_radius_qi[k]); - REQUIRE(isds_fortran[i].diag_diam_qi[k] == isds_cxx[i].diag_diam_qi[k]); - REQUIRE(isds_fortran[i].rho_qi[k] == isds_cxx[i].rho_qi[k]); - REQUIRE(isds_fortran[i].diag_equiv_reflectivity[k] == isds_cxx[i].diag_equiv_reflectivity[k]); - REQUIRE(isds_fortran[i].diag_eff_radius_qc[k] == isds_cxx[i].diag_eff_radius_qc[k]); - REQUIRE(isds_fortran[i].diag_eff_radius_qr[k] == isds_cxx[i].diag_eff_radius_qr[k]); + REQUIRE(isds_baseline[i].rho[k] == isds_cxx[i].rho[k]); + REQUIRE(isds_baseline[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); + REQUIRE(isds_baseline[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); + REQUIRE(isds_baseline[i].qv[k] == isds_cxx[i].qv[k]); + REQUIRE(isds_baseline[i].th_atm[k] == isds_cxx[i].th_atm[k]); + REQUIRE(isds_baseline[i].qc[k] == isds_cxx[i].qc[k]); + REQUIRE(isds_baseline[i].nc[k] == isds_cxx[i].nc[k]); + REQUIRE(isds_baseline[i].qr[k] == isds_cxx[i].qr[k]); + REQUIRE(isds_baseline[i].nr[k] == isds_cxx[i].nr[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].latent_heat_vapor[k] == latvap); + REQUIRE(isds_baseline[i].latent_heat_sublim[k] == latvap+latice); + REQUIRE(isds_baseline[i].mu_c[k] == isds_cxx[i].mu_c[k]); + REQUIRE(isds_baseline[i].nu[k] == isds_cxx[i].nu[k]); + REQUIRE(isds_baseline[i].lamc[k] == isds_cxx[i].lamc[k]); + REQUIRE(isds_baseline[i].mu_r[k] == isds_cxx[i].mu_r[k]); + REQUIRE(isds_baseline[i].lamr[k] == isds_cxx[i].lamr[k]); + REQUIRE(isds_baseline[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); + REQUIRE(isds_baseline[i].ze_rain[k] == isds_cxx[i].ze_rain[k]); + REQUIRE(isds_baseline[i].ze_ice[k] == isds_cxx[i].ze_ice[k]); + REQUIRE(isds_baseline[i].diag_vm_qi[k] == isds_cxx[i].diag_vm_qi[k]); + REQUIRE(isds_baseline[i].diag_eff_radius_qi[k] == isds_cxx[i].diag_eff_radius_qi[k]); + REQUIRE(isds_baseline[i].diag_diam_qi[k] == isds_cxx[i].diag_diam_qi[k]); + REQUIRE(isds_baseline[i].rho_qi[k] == isds_cxx[i].rho_qi[k]); + REQUIRE(isds_baseline[i].diag_equiv_reflectivity[k] == isds_cxx[i].diag_equiv_reflectivity[k]); + REQUIRE(isds_baseline[i].diag_eff_radius_qc[k] == isds_cxx[i].diag_eff_radius_qc[k]); + REQUIRE(isds_baseline[i].diag_eff_radius_qr[k] == isds_cxx[i].diag_eff_radius_qr[k]); } } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(fid); + } + } } void run_bfb_p3_main() { - auto engine = setup_random_test(); + auto engine = setup_random_test(3427727); - P3MainData isds_fortran[] = { + P3MainData isds_baseline[] = { // its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN P3MainData(1, 10, 1, 72, 1, 1.800E+03, false, true), P3MainData(1, 10, 1, 72, 1, 1.800E+03, true, false), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainData); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainData); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { d.randomize(engine, { {d.pres , {1.00000000E+02 , 9.87111111E+04}}, {d.dz , {1.22776609E+02 , 3.49039167E+04}}, @@ -409,16 +439,20 @@ void run_bfb_p3_main() }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainData isds_cxx[num_runs] = { - P3MainData(isds_fortran[0]), - P3MainData(isds_fortran[1]), + P3MainData(isds_baseline[0]), + P3MainData(isds_baseline[1]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/p3_main.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (auto& d : isds_baseline) { + d.read(fid); + } } // Get data from cxx @@ -434,11 +468,11 @@ void run_bfb_p3_main() d.template transpose(); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - const auto& df90 = isds_fortran[i]; - const auto& dcxx = isds_fortran[i]; - const auto tot = isds_fortran[i].total(df90.qc); + const auto& df90 = isds_baseline[i]; + const auto& dcxx = isds_baseline[i]; + const auto tot = isds_baseline[i].total(df90.qc); for (Int t = 0; t < tot; ++t) { REQUIRE(df90.qc[t] == dcxx.qc[t]); REQUIRE(df90.nc[t] == dcxx.nc[t]); @@ -474,6 +508,12 @@ void run_bfb_p3_main() REQUIRE(df90.precip_ice_surf[tot] == dcxx.precip_ice_surf[tot]); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(fid); + } + } } void run_bfb() From 85906748d0a90e489402f10f498ed1cfbd96c738 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 4 Nov 2024 08:46:13 -0700 Subject: [PATCH 218/366] Rename the LEV scream tag from 'Lev' to 'altitude' in the source grid when using a remap file. --- ...mxx_mam_microphysics_process_interface.cpp | 4 ++- .../mam/readfiles/tracer_reader_utils.hpp | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index b0ab232c255..feb87c18683 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -289,8 +289,10 @@ void MAMMicrophysics::set_grids( verti_emiss_cyclical_ymd); auto hor_rem = scream::mam_coupling::create_horiz_remapper( grid_, file_name, extfrc_map_file, var_names, data_tracer); + auto file_reader = - scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name); + scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name, + data_tracer, extfrc_map_file); VertEmissionsHorizInterp_.push_back(hor_rem); VertEmissionsDataReader_.push_back(file_reader); vert_emis_data_.push_back(data_tracer); diff --git a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp index 2e34db2b496..1f757e93027 100644 --- a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp @@ -448,6 +448,40 @@ inline std::shared_ptr create_tracer_data_reader( true); } // create_tracer_data_reader +inline std::shared_ptr create_tracer_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &tracer_data_file, + const TracerData &tracer_data, + const std::string &extfrc_map_file + ) { + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + + // NOTE: If we are using a vertical emission NC file with altitude instead of levels, + // we must rename this tag. This is only necessary when a map file is used. + bool rename_LEV_grid=false; + if(tracer_data.file_type == VERT_EMISSION && extfrc_map_file != ""){ + rename_LEV_grid=true; + } + + if (rename_LEV_grid) { + auto horiz_interp_src_grid = + io_grid->clone("tracer_horiz_interp_src_grid", true); + horiz_interp_src_grid->reset_field_tag_name(LEV, "altitude"); + horiz_interp_src_grid->reset_field_tag_name(ILEV, "altitude_int"); + return std::make_shared(tracer_data_file, horiz_interp_src_grid, io_fields, + true); + } else { + return std::make_shared(tracer_data_file, io_grid, io_fields, + true); + } + + +} // create_tracer_data_reader + inline void update_tracer_data_from_file( const std::shared_ptr &scorpio_reader, const int time_index, // zero-based From f1c9a3903b9aded1ccff7226b3b15f134c5066b6 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 4 Nov 2024 09:51:54 -0700 Subject: [PATCH 219/366] Removing unnecessary if statement. --- .../eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp index 1f757e93027..e31300a679a 100644 --- a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp @@ -462,12 +462,7 @@ inline std::shared_ptr create_tracer_data_reader( // NOTE: If we are using a vertical emission NC file with altitude instead of levels, // we must rename this tag. This is only necessary when a map file is used. - bool rename_LEV_grid=false; if(tracer_data.file_type == VERT_EMISSION && extfrc_map_file != ""){ - rename_LEV_grid=true; - } - - if (rename_LEV_grid) { auto horiz_interp_src_grid = io_grid->clone("tracer_horiz_interp_src_grid", true); horiz_interp_src_grid->reset_field_tag_name(LEV, "altitude"); @@ -479,7 +474,6 @@ inline std::shared_ptr create_tracer_data_reader( true); } - } // create_tracer_data_reader inline void update_tracer_data_from_file( From e166eaaeaffc472258ddd60c526af93a87dc6172 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 4 Nov 2024 11:44:57 -0700 Subject: [PATCH 220/366] All bfb unit tests complete --- .../p3/tests/p3_nc_conservation_tests.cpp | 40 +++++++++++------- .../p3/tests/p3_ni_conservation_tests.cpp | 39 ++++++++++------- .../p3/tests/p3_nr_conservation_tests.cpp | 42 ++++++++++++------- .../p3_prevent_liq_supersaturation_tests.cpp | 39 ++++++++++------- .../tests/p3_rain_imm_freezing_unit_tests.cpp | 18 ++++++-- .../tests/p3_rain_self_collection_tests.cpp | 18 ++++++-- ...p3_subgrid_variance_scaling_unit_tests.cpp | 29 +++++++++---- 7 files changed, 147 insertions(+), 78 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp index ecf47b99eaa..0161566d539 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp @@ -18,27 +18,31 @@ struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest: void run_bfb() { - auto engine = setup_random_test(); + auto engine = setup_random_test(1241556); - NcConservationData f90_data[max_pack_size]; + NcConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // Hold this fixed, this is not packed data + d.dt = baseline_data[0].dt; // Hold this fixed, this is not packed data } // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - nc_conservation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/nc_conservation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(fid); + } } // Get data from cxx. Run nc_conservation from a kernel and copy results back to host @@ -71,14 +75,20 @@ struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest: Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - NcConservationData& d_f90 = f90_data[i]; + NcConservationData& d_baseline = baseline_data[i]; NcConservationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.nc_collect_tend == d_cxx.nc_collect_tend); - REQUIRE(d_f90.nc2ni_immers_freeze_tend == d_cxx.nc2ni_immers_freeze_tend); - REQUIRE(d_f90.nc_accret_tend == d_cxx.nc_accret_tend); - REQUIRE(d_f90.nc2nr_autoconv_tend == d_cxx.nc2nr_autoconv_tend); + REQUIRE(d_baseline.nc_collect_tend == d_cxx.nc_collect_tend); + REQUIRE(d_baseline.nc2ni_immers_freeze_tend == d_cxx.nc2ni_immers_freeze_tend); + REQUIRE(d_baseline.nc_accret_tend == d_cxx.nc_accret_tend); + REQUIRE(d_baseline.nc2nr_autoconv_tend == d_cxx.nc2nr_autoconv_tend); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(fid); } } } // run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp index 41c13afd923..c5feff71ea8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp @@ -18,27 +18,31 @@ struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest: void run_bfb() { - auto engine = setup_random_test(); + auto engine = setup_random_test(1925985); - NiConservationData f90_data[max_pack_size]; + NiConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // hold dt fixed, it is not packed data + d.dt = baseline_data[0].dt; // hold dt fixed, it is not packed data } // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - ni_conservation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/ni_conservation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(fid); + } } // Get data from cxx. Run ni_conservation from a kernel and copy results back to host @@ -71,17 +75,22 @@ struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest: Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - NiConservationData& d_f90 = f90_data[i]; + NiConservationData& d_baseline = baseline_data[i]; NiConservationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.ni2nr_melt_tend == d_cxx.ni2nr_melt_tend); - REQUIRE(d_f90.ni_sublim_tend == d_cxx.ni_sublim_tend); - REQUIRE(d_f90.ni_selfcollect_tend == d_cxx.ni_selfcollect_tend); + REQUIRE(d_baseline.ni2nr_melt_tend == d_cxx.ni2nr_melt_tend); + REQUIRE(d_baseline.ni_sublim_tend == d_cxx.ni_sublim_tend); + REQUIRE(d_baseline.ni_selfcollect_tend == d_cxx.ni_selfcollect_tend); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(fid); } } } // run_bfb - }; } // namespace unit_test diff --git a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp index 2df8a01dbb3..c268a763d79 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp @@ -18,28 +18,32 @@ struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest: void run_bfb() { - auto engine = setup_random_test(); + auto engine = setup_random_test(912874); - NrConservationData f90_data[max_pack_size]; + NrConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // hold dt fixed, it is not packed data - d.nmltratio = f90_data[0].nmltratio; // hold nmltratio fixed, it is not packed data + d.dt = baseline_data[0].dt; // hold dt fixed, it is not packed data + d.nmltratio = baseline_data[0].nmltratio; // hold nmltratio fixed, it is not packed data } // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - nr_conservation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/nr_conservation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(fid); + } } // Get data from cxx. Run nr_conservation from a kernel and copy results back to host @@ -75,14 +79,20 @@ struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest: Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - NrConservationData& d_f90 = f90_data[i]; + NrConservationData& d_baseline = baseline_data[i]; NrConservationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.nr_collect_tend == d_cxx.nr_collect_tend); - REQUIRE(d_f90.nr2ni_immers_freeze_tend == d_cxx.nr2ni_immers_freeze_tend); - REQUIRE(d_f90.nr_selfcollect_tend == d_cxx.nr_selfcollect_tend); - REQUIRE(d_f90.nr_evap_tend == d_cxx.nr_evap_tend); + REQUIRE(d_baseline.nr_collect_tend == d_cxx.nr_collect_tend); + REQUIRE(d_baseline.nr2ni_immers_freeze_tend == d_cxx.nr2ni_immers_freeze_tend); + REQUIRE(d_baseline.nr_selfcollect_tend == d_cxx.nr_selfcollect_tend); + REQUIRE(d_baseline.nr_evap_tend == d_cxx.nr_evap_tend); + } + } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(fid); } } } // run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index 2b845e9c97b..acd7e7d7454 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -93,18 +93,18 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = setup_random_test(1298758); - PreventLiqSupersaturationData f90_data[max_pack_size]; + PreventLiqSupersaturationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // Hold this fixed, this is not packed data + d.dt = baseline_data[0].dt; // Hold this fixed, this is not packed data // C++ impl uses constants for latent_heat values. Manually set here - // so F90 can match + // so BASELINE can match d.latent_heat_vapor = latvap; d.latent_heat_sublim = latvap+latice; } @@ -113,7 +113,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U // fortran calls so that inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); // Save copy of inout vars to check that prevent_liq_supersaturation always makes them smaller @@ -123,9 +123,13 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U qr2qv_evap_tend_init[i] = cxx_host(i).qr2qv_evap_tend; } - // Get data from fortran - for (auto& d : f90_data) { - prevent_liq_supersaturation(d); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/prevent_liq_supersaturation.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(fid); + } } // Get data from cxx. Run prevent_liq_supersaturation from a kernel and copy results back to host @@ -158,13 +162,13 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U Kokkos::deep_copy(cxx_host, cxx_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { // Verify BFB results - PreventLiqSupersaturationData& d_f90 = f90_data[i]; + PreventLiqSupersaturationData& d_baseline = baseline_data[i]; PreventLiqSupersaturationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); - REQUIRE(d_f90.qr2qv_evap_tend == d_cxx.qr2qv_evap_tend); + REQUIRE(d_baseline.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); + REQUIRE(d_baseline.qr2qv_evap_tend == d_cxx.qr2qv_evap_tend); //Verify tendencies are always >=0: REQUIRE(d_cxx.qi2qv_sublim_tend>=0); @@ -175,8 +179,13 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U REQUIRE(d_cxx.qr2qv_evap_tend<=qr2qv_evap_tend_init[i]); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(fid); + } + } } // run_bfb - }; } // namespace unit_test diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp index e19b83d6c2a..92123d63e82 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp @@ -69,9 +69,13 @@ void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - rain_immersion_freezing(rain_imm_freezing_data[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/rain_immersion_freezing.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + rain_imm_freezing_data[i].read(fid); + } } // Run the lookup from a kernel and copy results back to host @@ -106,12 +110,18 @@ void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(rain_imm_freezing_data[s].qr2qi_immers_freeze_tend == host_data[s].qr2qi_immers_freeze_tend); REQUIRE(rain_imm_freezing_data[s].nr2ni_immers_freeze_tend == host_data[s].nr2ni_immers_freeze_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(fid); + } + } } }; diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp index 7bcdf59dac8..d9d8ef00412 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp @@ -56,9 +56,13 @@ struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest std::copy(&dc[0], &dc[0] + max_pack_size, dc_host.data()); Kokkos::deep_copy(dc_device, dc_host); - //Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - rain_self_collection(dc[i]); + // Read baseline data + std::string baseline_name = this->m_baseline_path + "/rain_self_collection.dat"; + if (this->m_baseline_action == COMPARE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + for (Int i = 0; i < max_pack_size; ++i) { + dc[i].read(fid); + } } //Run function from a kernal and copy results back to the host @@ -91,7 +95,7 @@ struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest Kokkos::deep_copy(dc_host, dc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(dc[s].rho == dc_host(s).rho); REQUIRE(dc[s].qr_incld == dc_host(s).qr_incld); @@ -99,6 +103,12 @@ struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest REQUIRE(dc[s].nr_selfcollect_tend == dc_host(s).nr_selfcollect_tend); } } + else if (this->m_baseline_action == GENERATE) { + auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + for (Int s = 0; s < max_pack_size; ++s) { + dc_host(s).write(fid); + } + } } void run_bfb() { diff --git a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp index 9e75889df0c..a71b3abd422 100644 --- a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp @@ -36,22 +36,30 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::Un //Set of exponents to loop over Scalar expons[3] = {1.0,2.47,0.1}; - //initialize struct required for F90 call - SubgridVarianceScalingData f_data; - Scalar f_scaling; + Scalar baseline_scaling; //Make C++ output available on host and device view_1d scaling_device("c scaling",1); auto scaling_host = Kokkos::create_mirror_view(scaling_device); + std::string baseline_name = this->m_baseline_path + "/subgrid_variance_scaling.dat"; + ekat::FILEPtr rfid = nullptr; + ekat::FILEPtr wfid = nullptr; + if (this->m_baseline_action == COMPARE) { + rfid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + } + else if (this->m_baseline_action == GENERATE) { + wfid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + } + for (Int i = 0; i < 3; ++i) { // loop over exponents for (Int j = 0; j < 16; ++j) { // loop over relvars - // Get F90 solution + // Get baseline solution // ---------------------------------- - f_data.relvar=relvars[j]; - f_data.expon =expons[i]; - f_scaling = subgrid_variance_scaling(f_data); + if (this->m_baseline_action == COMPARE) { + ekat::read(&baseline_scaling, 1, rfid); + } // Get C++ solution // ---------------------------------- @@ -72,8 +80,11 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::Un Kokkos::deep_copy(scaling_host, scaling_device); // Validate results - if (SCREAM_BFB_TESTING) { - REQUIRE(f_scaling == scaling_host(0) ); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + REQUIRE(baseline_scaling == scaling_host(0) ); + } + else if (this->m_baseline_action == GENERATE) { + ekat::write(&scaling_host(0), 1, wfid); } } //end loop over relvar[j] } //end loop over expons[i] From 23fd73091a0c1cd214289c7e7c89276be179b769 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 4 Nov 2024 12:40:18 -0700 Subject: [PATCH 221/366] Fix bug by using target grid instead of source grid. --- .../mam/eamxx_mam_microphysics_process_interface.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index feb87c18683..0b1440b3826 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -205,7 +205,7 @@ void MAMMicrophysics::set_grids( LinozHorizInterp_, linoz_file_name_); // linoz reader - const auto io_grid_linoz = LinozHorizInterp_->get_src_grid(); + const auto io_grid_linoz = LinozHorizInterp_->get_tgt_grid(); const int num_cols_io_linoz = io_grid_linoz->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_linoz = @@ -233,7 +233,7 @@ void MAMMicrophysics::set_grids( TracerHorizInterp_, oxid_file_name_); const int nvars = int(var_names.size()); - const auto io_grid = TracerHorizInterp_->get_src_grid(); + const auto io_grid = TracerHorizInterp_->get_tgt_grid(); const int num_cols_io = io_grid->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io = @@ -308,7 +308,7 @@ void MAMMicrophysics::set_grids( // I am assuming the order of species in extfrc_lst_. // Indexing in mam4xx is fortran. forcings_[i].frc_ndx = i + 1; - const auto io_grid_emis = VertEmissionsHorizInterp_[i]->get_src_grid(); + const auto io_grid_emis = VertEmissionsHorizInterp_[i]->get_tgt_grid(); const int num_cols_io_emis = io_grid_emis->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_emis = From f72cc87ef0e493dae0f26ce31ba2898a1c77eaf0 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Mon, 4 Nov 2024 12:04:50 -0800 Subject: [PATCH 222/366] Add ilev as a default output in all EAMxx output streams This commit follows the example for the geometry data for 'lev' and adds 'ilev'. The variable 'ilev' will now be a default variable in all EAMxx output similar to what is currently done with lat/lon, hyam/hybm, hyai/hybi and lev. Addresses #3022 --- .../dynamics/homme/homme_grids_manager.cpp | 18 +++++-- .../share/grid/mesh_free_grids_manager.cpp | 48 ++++++++++++++----- .../eamxx/src/share/io/scream_io_utils.hpp | 5 +- .../src/share/io/scream_output_manager.cpp | 2 +- 4 files changed, 54 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index 87009c7d074..df5de6827f6 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -271,6 +271,7 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { auto hyam = phys_grid->create_geometry_data("hyam",layout_mid,nondim); auto hybm = phys_grid->create_geometry_data("hybm",layout_mid,nondim); auto lev = phys_grid->create_geometry_data("lev", layout_mid,mbar); + auto ilev = phys_grid->create_geometry_data("ilev",layout_int,mbar); for (auto f : {hyai, hybi, hyam, hybm}) { auto f_d = get_grid("Dynamics")->get_geometry_data(f.name()); @@ -281,13 +282,20 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { // Build lev from hyam and hybm const Real ps0 = 100000.0; - auto hya_v = hyam.get_view(); - auto hyb_v = hybm.get_view(); - auto lev_v = lev.get_view(); - for (int ii=0;iiget_num_vertical_levels();ii++) { - lev_v(ii) = 0.01*ps0*(hya_v(ii)+hyb_v(ii)); + auto hyam_v = hyam.get_view(); + auto hybm_v = hybm.get_view(); + auto hyai_v = hyai.get_view(); + auto hybi_v = hybi.get_view(); + auto lev_v = lev.get_view(); + auto ilev_v = ilev.get_view(); + auto num_v_levs = phys_grid->get_num_vertical_levels(); + for (int ii=0;iiget_num_vertical_levels()}); + FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()}); const auto units = ekat::units::Units::nondimensional(); auto lat = grid->create_geometry_data("lat" , grid->get_2d_scalar_layout(), units); auto lon = grid->create_geometry_data("lon" , grid->get_2d_scalar_layout(), units); auto hyam = grid->create_geometry_data("hyam" , layout_mid, units); auto hybm = grid->create_geometry_data("hybm" , layout_mid, units); + auto hyai = grid->create_geometry_data("hyai" , layout_int, units); + auto hybi = grid->create_geometry_data("hybi" , layout_int, units); auto lev = grid->create_geometry_data("lev" , layout_mid, units); + auto ilev = grid->create_geometry_data("ilev" , layout_int, units); lat.deep_copy(ekat::ScalarTraits::invalid()); lon.deep_copy(ekat::ScalarTraits::invalid()); hyam.deep_copy(ekat::ScalarTraits::invalid()); hybm.deep_copy(ekat::ScalarTraits::invalid()); lev.deep_copy(ekat::ScalarTraits::invalid()); + ilev.deep_copy(ekat::ScalarTraits::invalid()); lat.sync_to_dev(); lon.sync_to_dev(); hyam.sync_to_dev(); hybm.sync_to_dev(); lev.sync_to_dev(); + ilev.sync_to_dev(); } else if (geo_data_source=="IC_FILE"){ const auto& filename = m_params.get("ic_filename"); if (scorpio::has_var(filename,"lat") && @@ -173,7 +179,9 @@ add_geo_data (const nonconstgrid_ptr_type& grid) const } if (scorpio::has_var(filename,"hyam") && - scorpio::has_var(filename,"hybm")) { + scorpio::has_var(filename,"hybm") && + scorpio::has_var(filename,"hyai") && + scorpio::has_var(filename,"hybi") ) { load_vertical_coordinates(grid,filename); } } @@ -234,53 +242,71 @@ load_vertical_coordinates (const nonconstgrid_ptr_type& grid, const std::string& using namespace ekat::units; FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); + FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()}); Units nondim = Units::nondimensional(); Units mbar (100*Pa,"mb"); auto hyam = grid->create_geometry_data("hyam", layout_mid, nondim); auto hybm = grid->create_geometry_data("hybm", layout_mid, nondim); + auto hyai = grid->create_geometry_data("hyai", layout_int, nondim); + auto hybi = grid->create_geometry_data("hybi", layout_int, nondim); auto lev = grid->create_geometry_data("lev", layout_mid, mbar); + auto ilev = grid->create_geometry_data("ilev", layout_int, mbar); // Create host mirrors for reading in data std::map host_views = { { "hyam", hyam.get_view() }, - { "hybm", hybm.get_view() } + { "hybm", hybm.get_view() }, + { "hyai", hyai.get_view() }, + { "hybi", hybi.get_view() } }; // Store view layouts using namespace ShortFieldTagsNames; std::map layouts = { { "hyam", hyam.get_header().get_identifier().get_layout() }, - { "hybm", hybm.get_header().get_identifier().get_layout() } + { "hybm", hybm.get_header().get_identifier().get_layout() }, + { "hyai", hyai.get_header().get_identifier().get_layout() }, + { "hybi", hybi.get_header().get_identifier().get_layout() } }; // Read hyam/hybm into host views ekat::ParameterList vcoord_reader_pl; vcoord_reader_pl.set("Filename",filename); - vcoord_reader_pl.set>("Field Names",{"hyam","hybm"}); + vcoord_reader_pl.set>("Field Names",{"hyam","hybm","hyai","hybi"}); AtmosphereInput vcoord_reader(vcoord_reader_pl,grid, host_views, layouts); vcoord_reader.read_variables(); vcoord_reader.finalize(); - // Build lev from hyam and hybm + // Build lev and ilev from hyam and hybm, and ilev from hyai and hybi using PC = scream::physics::Constants; const Real ps0 = PC::P0; - auto hya_v = hyam.get_view(); - auto hyb_v = hybm.get_view(); - auto lev_v = lev.get_view(); - for (int ii=0;iiget_num_vertical_levels();ii++) { - lev_v(ii) = 0.01*ps0*(hya_v(ii)+hyb_v(ii)); + auto hyam_v = hyam.get_view(); + auto hybm_v = hybm.get_view(); + auto lev_v = lev.get_view(); + auto hyai_v = hyai.get_view(); + auto hybi_v = hybi.get_view(); + auto ilev_v = ilev.get_view(); + auto num_lev = grid->get_num_vertical_levels(); + for (int ii=0;iinum_lev;ii++) { + lev_v(ii) = 0.01*ps0*(hyam_v(ii)+hybm_v(ii)); + ilev_v(ii) = 0.01*ps0*(hyai_v(ii)+hybi_v(ii)); } + // Note, ilev is just 1 more level than the number of midpoint levs + ilev_v(num_lev) = 0.01*ps0*(hyai_v(num_lev)+hybi_v(num_lev)); // Sync to dev hyam.sync_to_dev(); hybm.sync_to_dev(); + hyai.sync_to_dev(); + hybi.sync_to_dev(); lev.sync_to_dev(); + ilev.sync_to_dev(); #ifndef NDEBUG - for (auto f : {hyam, hybm}) { + for (auto f : {hyam, hybm, hyai, hybi}) { auto nan_check = std::make_shared(f,grid)->check(); EKAT_REQUIRE_MSG (nan_check.result==CheckResult::Pass, "ERROR! NaN values detected in " + f.name() + " field.\n" + nan_check.msg); diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 01bc46e1e60..d6c36ab3758 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -85,8 +85,9 @@ struct LongNames { // Create map of longnames, can be added to as developers see fit. std::map name_2_longname = { - {"lev","hybrid level at midpoints (1000*(A+B))"}, - {"hyai","hybrid A coefficient at layer interfaces"}, + {"lev","hybrid level at midpoints (1000*(A+B))"}, + {"ilev","hybrid level at interfaces (1000*(A+B))"}, + {"hyai","hybrid A coefficient at layer interfaces"}, {"hybi","hybrid B coefficient at layer interfaces"}, {"hyam","hybrid A coefficient at layer midpoints"}, {"hybm","hybrid B coefficient at layer midpoints"} diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 0628a889be9..4ac141c42e1 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -402,7 +402,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) // Check if we need to open a new file if (not filespecs.is_open) { filespecs.filename = compute_filename (filespecs,timestamp); - // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm) + // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm/hyai/hybi) setup_file(filespecs,control); } From a2b0af0ae7df8a91ce9aaf364961f1bb004c261f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 4 Nov 2024 13:08:03 -0700 Subject: [PATCH 223/366] p3_run_and_cmp fixed up --- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 42 +++++----- .../src/physics/p3/tests/p3_run_and_cmp.cpp | 78 +++++++++++-------- 2 files changed, 66 insertions(+), 54 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index d5980d91cbf..66346279018 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -62,13 +62,15 @@ CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" LABELS "p3;physics") # Make sure that a diff in the two implementation triggers a failed test (in debug only) -CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3 - EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3;physics;fail" - ${FORCE_RUN_DIFF_FAILS}) +if (SCREAM_ENABLE_BASELINE_TESTS) + CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp + LIBS p3 + EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3;physics;fail" + ${FORCE_RUN_DIFF_FAILS}) +endif() if (NOT SCREAM_P3_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" @@ -87,24 +89,24 @@ if (NOT SCREAM_P3_SMALL_KERNELS) ${FORCE_RUN_DIFF_FAILS}) endif() -CreateUnitTestExec(p3_run_and_cmp "p3_run_and_cmp.cpp" +CreateUnitTest(p3_run_and_cmp "p3_run_and_cmp.cpp" LIBS p3 - EXCLUDE_MAIN_CPP) - -CreateUnitTestFromExec(p3_run_and_cmp_cxx p3_run_and_cmp + EXCLUDE_MAIN_CPP THREADS ${SCREAM_TEST_MAX_THREADS} EXE_ARGS "${BASELINE_FILE_ARG}" LABELS "p3;physics") # Make sure that a diff from baselines triggers a failed test (in debug only) -CreateUnitTest(p3_run_and_cmp_cxx_fail "p3_run_and_cmp.cpp" - LIBS p3 - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "p3;physics;fail" - EXCLUDE_MAIN_CPP - ${FORCE_RUN_DIFF_FAILS}) +if (SCREAM_ENABLE_BASELINE_TESTS) + CreateUnitTest(p3_run_and_cmp_fail "p3_run_and_cmp.cpp" + LIBS p3 + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "p3;physics;fail" + EXCLUDE_MAIN_CPP + ${FORCE_RUN_DIFF_FAILS}) +endif() # Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it if (SCREAM_TEST_MAX_THREADS GREATER 1) @@ -112,4 +114,4 @@ if (SCREAM_TEST_MAX_THREADS GREATER 1) set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) endif() -set_tests_properties (p3_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") +set_tests_properties (p3_run_and_cmp${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index 5bc12b2464d..9e491cea6a4 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -75,7 +75,7 @@ struct Baseline { } } - Int generate_baseline (const std::string& filename, bool use_fortran) { + Int generate_baseline (const std::string& filename) { auto fid = ekat::FILEPtr(fopen(filename.c_str(), "w")); EKAT_REQUIRE_MSG( fid, "generate_baseline can't write " << filename); Int nerr = 0; @@ -93,16 +93,13 @@ struct Baseline { std::cout << "Running P3 with ni=" << d->ncol << ", nk=" << d->nlev << ", dt=" << d->dt << ", ts=" << d->it << ", predict_nc=" << d->do_predict_nc - << ", prescribed_CCN=" << d->do_prescribed_CCN; - - if (!use_fortran) { - std::cout << ", small_packn=" << SCREAM_SMALL_PACK_SIZE; - } - std::cout << std::endl; + << ", prescribed_CCN=" << d->do_prescribed_CCN + << ", small_packn=" << SCREAM_SMALL_PACK_SIZE + << std::endl; } for (int it=0; it 0) { // do not count the "cold" run total_duration_microsec += current_microsec; @@ -123,29 +120,40 @@ struct Baseline { return nerr; } - Int run_and_cmp (const std::string& filename, const double& tol, bool use_fortran) { + Int run_and_cmp (const std::string& filename, const double& tol, bool no_baseline) { auto fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); Int nerr = 0, ne; int case_num = 0; for (auto ps : params_) { case_num++; - // Read the reference impl's data from the baseline file. - const auto d_ref = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); - set_params(ps, *d_ref); - // Now run a sequence of other impls. This includes the reference - // implementation b/c it's likely we'll want to change it as we go. - { + if (no_baseline) { const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); set_params(ps, *d); p3_init(); for (int it=0; it Path to directory containing baselines.\n" " -t Tolerance for relative error. Default 0.\n" " -s Number of timesteps. Default=6.\n" " -dt Length of timestep. Default=300.\n" " -i Number of columns. Default=3.\n" " -k Number of vertical levels. Default=72.\n" " -r Number of repetitions, implies timing run (generate + no I/O). Default=0.\n" - " -p yes|no|both. Default=both.\n" - " -c yes|no|both. Default=both.\n"; + " --predict-nc yes|no|both. Default=both.\n" + " --prescribed-ccn yes|no|both. Default=both.\n"; return 1; } - bool generate = false, use_fortran = false; + bool generate = false, no_baseline = true; scream::Real tol = SCREAM_BFB_TESTING ? 0 : std::numeric_limits::infinity(); Int timesteps = 6; Int dt = 300; @@ -247,8 +257,8 @@ int main (int argc, char** argv) { std::string prescribed_ccn = "both"; std::string baseline_fn; for (int i = 1; i < argc-1; ++i) { - if (ekat::argv_matches(argv[i], "-g", "--generate")) generate = true; - if (ekat::argv_matches(argv[i], "-f", "--fortran")) use_fortran = true; + if (ekat::argv_matches(argv[i], "-g", "--generate")) { generate = true; no_baseline = false; } + if (ekat::argv_matches(argv[i], "-c", "--compare")) { no_baseline = false; } if (ekat::argv_matches(argv[i], "-t", "--tol")) { expect_another_arg(i, argc); ++i; @@ -292,14 +302,14 @@ int main (int argc, char** argv) { generate = true; } } - if (ekat::argv_matches(argv[i], "-p", "--predict-nc")) { + if (ekat::argv_matches(argv[i], "-pn", "--predict-nc")) { expect_another_arg(i, argc); ++i; predict_nc = std::string(argv[i]); EKAT_REQUIRE_MSG(predict_nc == "yes" || predict_nc == "no" || predict_nc == "both", "Predict option value must be one of yes|no|both"); } - if (ekat::argv_matches(argv[i], "-c", "--prescribed-ccn")) { + if (ekat::argv_matches(argv[i], "-pc", "--prescribed-ccn")) { expect_another_arg(i, argc); ++i; prescribed_ccn = std::string(argv[i]); @@ -308,8 +318,8 @@ int main (int argc, char** argv) { } } - // Decorate baseline name with precision. - baseline_fn += std::to_string(sizeof(scream::Real)); + // Compute full baseline file name with precision. + baseline_fn += "/p3_run_and_cmp.baseline" + std::to_string(sizeof(scream::Real)); std::vector args; for (int i=0; i(dt), ncol, nlev, repeat, predict_nc, prescribed_ccn); if (generate) { std::cout << "Generating to " << baseline_fn << "\n"; - nerr += bln.generate_baseline(baseline_fn, use_fortran); + nerr += bln.generate_baseline(baseline_fn); } else { printf("Comparing with %s at tol %1.1e\n", baseline_fn.c_str(), tol); - nerr += bln.run_and_cmp(baseline_fn, tol, use_fortran); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); } P3GlobalForFortran::deinit(); } scream::finalize_scream_session(); From 5e46e3373a0c246ec081a1c2142a6548a05be3a3 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 4 Nov 2024 13:37:43 -0700 Subject: [PATCH 224/366] Progress. Can not remove fortran --- .../p3/tests/p3_cloud_sed_unit_tests.cpp | 2 +- .../p3/tests/p3_evaporate_rain_unit_tests.cpp | 2 +- .../p3_ice_deposition_sublimation_tests.cpp | 2 +- .../p3/tests/p3_ice_sed_unit_tests.cpp | 6 +-- .../p3_ice_supersat_conservation_tests.cpp | 2 +- .../p3/tests/p3_nc_conservation_tests.cpp | 2 +- .../p3/tests/p3_ni_conservation_tests.cpp | 2 +- .../p3/tests/p3_nr_conservation_tests.cpp | 2 +- .../p3_prevent_liq_supersaturation_tests.cpp | 2 +- .../p3/tests/p3_rain_sed_unit_tests.cpp | 40 +++++++++---------- .../src/physics/p3/tests/p3_unit_tests.cpp | 6 +-- 11 files changed, 34 insertions(+), 34 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index ace2523fc2d..29fbdad8aef 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -46,7 +46,7 @@ void run_bfb() d.randomize(engine, { {d.qc_incld, {C::QSMALL/2, C::QSMALL*2}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CloudSedData csds_cxx[num_runs] = { CloudSedData(csds_baseline[0]), diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index c5626e0387c..a17165bafdd 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -132,7 +132,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest: constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - //fortran generated data is input to the following + //baseline generated data is input to the following //This subroutine has 20 args, only 18 are supplied here for invoking it as last 2 are intent-outs //note that dt is the same val for each row - this is needed since dt is a scalar and all rows are executed simultaneously on CPU in C++. //row1: above freezing, should trigger diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp index b359ad661e9..7a52c1b42f0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp @@ -74,7 +74,7 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un // d.randomize(); //} - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index 3afcc6af92b..f63e0badfb0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -71,7 +71,7 @@ void run_bfb_calc_bulk_rhime() {5.164017E-10, 0.000000E+00, 0.000000E+00}, }; - // Sync to device, needs to happen before fortran calls so that + // Sync to device, needs to happen before reads so that // inout data is in original state view_1d cbrr_device("cbrr", max_pack_size); const auto cbrr_host = Kokkos::create_mirror_view(cbrr_device); @@ -150,7 +150,7 @@ void run_bfb_ice_sed() d.randomize(engine, { {d.qi_incld, {C::QSMALL/2, C::QSMALL*2}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state IceSedData isds_cxx[num_runs] = { IceSedData(isds_baseline[0]), @@ -233,7 +233,7 @@ void run_bfb_homogeneous_freezing() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state HomogeneousFreezingData hfds_cxx[num_runs] = { HomogeneousFreezingData(hfds_baseline[0]), diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp index e71463f1e25..c15c1383ab7 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp @@ -36,7 +36,7 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::Uni d.latent_heat_sublim = latvap+latice; } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); diff --git a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp index 0161566d539..e13d49affc0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp @@ -29,7 +29,7 @@ struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest: d.dt = baseline_data[0].dt; // Hold this fixed, this is not packed data } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); diff --git a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp index c5feff71ea8..2f2db25d1d7 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp @@ -29,7 +29,7 @@ struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest: d.dt = baseline_data[0].dt; // hold dt fixed, it is not packed data } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); diff --git a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp index c268a763d79..5b8b0d83951 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp @@ -30,7 +30,7 @@ struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest: d.nmltratio = baseline_data[0].nmltratio; // hold nmltratio fixed, it is not packed data } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index acd7e7d7454..c4f95b28c83 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -110,7 +110,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U } // Create copies of data for use by cxx and sync it to device. Needs to happen before - // fortran calls so that inout data is in original state + // reads so that inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index 22f0c50b41d..abad922cfb4 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -147,7 +147,7 @@ void run_bfb_rain_sed() constexpr Scalar dt = 1.800E+03; #endif - RainSedData rsds_fortran[] = { + RainSedData rsds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, precip_liq_surf RainSedData(1, 72, 27, 72, -1, dt, 1/dt, 0.0), RainSedData(1, 72, 72, 27, 1, dt, 1/dt, 1.0), @@ -155,27 +155,27 @@ void run_bfb_rain_sed() RainSedData(1, 72, 27, 27, 1, dt, 1/dt, 2.0), }; - static constexpr Int num_runs = sizeof(rsds_fortran) / sizeof(RainSedData); + static constexpr Int num_runs = sizeof(rsds_baseline) / sizeof(RainSedData); // Set up random input data - for (auto& d : rsds_fortran) { + for (auto& d : rsds_baseline) { d.randomize(engine, { {d.qr_incld, {C::QSMALL/2, C::QSMALL*2}} }); } // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state RainSedData rsds_cxx[num_runs] = { - RainSedData(rsds_fortran[0]), - RainSedData(rsds_fortran[1]), - RainSedData(rsds_fortran[2]), - RainSedData(rsds_fortran[3]), + RainSedData(rsds_baseline[0]), + RainSedData(rsds_baseline[1]), + RainSedData(rsds_baseline[2]), + RainSedData(rsds_baseline[3]), }; // Read baseline data std::string baseline_name = this->m_baseline_path + "/rain_sed.dat"; if (this->m_baseline_action == COMPARE) { auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); - for (auto& d : rsds_fortran) { + for (auto& d : rsds_baseline) { d.read(fid); } } @@ -198,20 +198,20 @@ void run_bfb_rain_sed() if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(rsds_fortran[i].kbot, rsds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(rsds_fortran[i].kbot, rsds_fortran[i].ktop); // 0-based indx + Int start = std::min(rsds_baseline[i].kbot, rsds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(rsds_baseline[i].kbot, rsds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(rsds_fortran[i].qr[k] == rsds_cxx[i].qr[k]); - REQUIRE(rsds_fortran[i].nr[k] == rsds_cxx[i].nr[k]); - REQUIRE(rsds_fortran[i].nr_incld[k] == rsds_cxx[i].nr_incld[k]); - REQUIRE(rsds_fortran[i].mu_r[k] == rsds_cxx[i].mu_r[k]); - REQUIRE(rsds_fortran[i].lamr[k] == rsds_cxx[i].lamr[k]); - REQUIRE(rsds_fortran[i].precip_liq_flux[k] == rsds_cxx[i].precip_liq_flux[k]); - REQUIRE(rsds_fortran[i].qr_tend[k] == rsds_cxx[i].qr_tend[k]); - REQUIRE(rsds_fortran[i].nr_tend[k] == rsds_cxx[i].nr_tend[k]); + REQUIRE(rsds_baseline[i].qr[k] == rsds_cxx[i].qr[k]); + REQUIRE(rsds_baseline[i].nr[k] == rsds_cxx[i].nr[k]); + REQUIRE(rsds_baseline[i].nr_incld[k] == rsds_cxx[i].nr_incld[k]); + REQUIRE(rsds_baseline[i].mu_r[k] == rsds_cxx[i].mu_r[k]); + REQUIRE(rsds_baseline[i].lamr[k] == rsds_cxx[i].lamr[k]); + REQUIRE(rsds_baseline[i].precip_liq_flux[k] == rsds_cxx[i].precip_liq_flux[k]); + REQUIRE(rsds_baseline[i].qr_tend[k] == rsds_cxx[i].qr_tend[k]); + REQUIRE(rsds_baseline[i].nr_tend[k] == rsds_cxx[i].nr_tend[k]); } - REQUIRE(rsds_fortran[i].precip_liq_flux[end] == rsds_cxx[i].precip_liq_flux[end]); - REQUIRE(rsds_fortran[i].precip_liq_surf == rsds_cxx[i].precip_liq_surf); + REQUIRE(rsds_baseline[i].precip_liq_flux[end] == rsds_cxx[i].precip_liq_flux[end]); + REQUIRE(rsds_baseline[i].precip_liq_surf == rsds_cxx[i].precip_liq_surf); } } else if (this->m_baseline_action == GENERATE) { diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index 9bf22760279..2c1971f0ba9 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -513,7 +513,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitT constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - //fortran generated data is input to the following + //baseline generated data is input to the following P3UpdatePrognosticIceData pupidc[max_pack_size] = { {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, @@ -748,7 +748,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::U constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - //fortran generated data is input to the following + //baseline generated data is input to the following GetTimeSpacePhysVarsData gtspvd[max_pack_size] = { // T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i {2.9792E+02, 9.8711E+04, 1.1532E+00, latvap, latvap+latice, 2.0321E-02, 2.0321E-02}, @@ -870,7 +870,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitT void update_prognostic_liquid_unit_bfb_tests() { constexpr Scalar latvap = C::LatVap; - //fortran generated data is input to the following + //baseline generated data is input to the following P3UpdatePrognosticLiqData pupldc[max_pack_size] = { {1.0631E-12, 1.0631E+00, 1.5833E-12, 1.5833E+00, 2.4190E-02, 0.0000E+00, 0.0000E+00, 0.0000E+00, 4.2517E+00, From 58b70ac410615b646d8cd83161b1792512e56b51 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 4 Nov 2024 15:29:00 -0700 Subject: [PATCH 225/366] Remove most of the f90 bridges --- .../eamxx/src/physics/p3/p3_functions_f90.cpp | 671 +------------- .../eamxx/src/physics/p3/p3_functions_f90.hpp | 144 +-- components/eamxx/src/physics/p3/p3_iso_c.f90 | 821 ------------------ .../eamxx/src/physics/p3/p3_main_wrap.cpp | 58 +- .../eamxx/src/physics/p3/p3_main_wrap.hpp | 5 +- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 12 +- .../p3_back_to_cell_average_unit_tests.cpp | 4 - .../p3/tests/p3_check_values_unit_tests.cpp | 2 +- .../p3/tests/p3_cloud_sed_unit_tests.cpp | 2 +- .../p3/tests/p3_ice_sed_unit_tests.cpp | 4 +- .../physics/p3/tests/p3_main_unit_tests.cpp | 8 +- .../p3/tests/p3_rain_sed_unit_tests.cpp | 2 +- .../eamxx/src/physics/p3/tests/p3_tests.cpp | 7 +- .../physics/p3/tests/p3_upwind_unit_tests.cpp | 4 +- 14 files changed, 101 insertions(+), 1643 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index b5e051a4d11..40d5342a31f 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -11,275 +11,22 @@ using scream::Real; using scream::Int; -// -// A C++ interface to micro_p3 fortran calls and vice versa -// extern "C" { void p3_init_a_c(Real* ice_table_vals, Real* collect_table_vals); -void find_lookuptable_indices_1a_c(Int* dumi, Int* dumjj, Int* dumii, Int* dumzz, - Real* dum1, Real* dum4, Real* dum5, Real* dum6, - Real qi, Real ni, Real qm, Real rhop); - -void find_lookuptable_indices_1b_c(Int* dumj, Real* dum3, Real qr, Real nr); - -void access_lookup_table_c(Int dumjj, Int dumii, Int dumi, Int index, - Real dum1, Real dum4, Real dum5, Real* proc); - -void access_lookup_table_coll_c(Int dumjj, Int dumii, Int dumj, Int dumi, Int index, - Real dum1, Real dum3, Real dum4, Real dum5, Real* proc); - -void back_to_cell_average_c(Real cld_frac_l_, Real cld_frac_r_, Real cld_frac_i_, - Real* qc2qr_accret_tend_, Real* qr2qv_evap_tend_, Real* qc2qr_autoconv_tend_, - Real* nc_accret_tend_, Real* nc_selfcollect_tend_, Real* nc2nr_autoconv_tend_, - Real* nr_selfcollect_tend_, Real* nr_evap_tend_, Real* ncautr_, - Real* qi2qv_sublim_tend_, - Real* nr_ice_shed_tend_, Real* qc2qi_hetero_freeze_tend_, Real* qr2qi_collect_tend_, - Real* qc2qr_ice_shed_tend_, Real* qi2qr_melt_tend_, Real* qc2qi_collect_tend_, - Real* qr2qi_immers_freeze_tend_, Real* ni2nr_melt_tend_, Real* nc_collect_tend_, - Real* ncshdc_, Real* nc2ni_immers_freeze_tend_, Real* nr_collect_tend_, - Real* ni_selfcollect_tend_, Real* qv2qi_vapdep_tend_, Real* nr2ni_immers_freeze_tend_, - Real* ni_sublim_tend_, Real* qv2qi_nucleat_tend_, Real* ni_nucleat_tend_, - Real* qc2qi_berg_tend_); - -void cloud_water_conservation_c(Real qc, Real dt, Real* qc2qr_autoconv_tend, Real* qc2qr_accret_tend, Real* qc2qi_collect_tend, - Real* qc2qi_hetero_freeze_tend, Real* qc2qr_ice_shed_tend, Real* qc2qi_berg_tend, Real* qi2qv_sublim_tend, Real* qv2qi_vapdep_tend); - -void rain_water_conservation_c(Real qr, Real qc2qr_autoconv_tend, Real qc2qr_accret_tend, Real qi2qr_melt_tend, Real qc2qr_ice_shed_tend, - Real dt, Real* qr2qv_evap_tend, Real* qr2qi_collect_tend, Real* qr2qi_immers_freeze_tend); - -void ice_water_conservation_c(Real qi, Real qv2qi_vapdep_tend, Real qv2qi_nucleat_tend, Real qc2qi_berg_tend, Real qr2qi_collect_tend, Real qc2qi_collect_tend, - Real qr2qi_immers_freeze_tend, Real qc2qi_hetero_freeze_tend, Real dt, Real* qi2qv_sublim_tend, Real* qi2qr_melt_tend); - -void get_cloud_dsd2_c(Real qc, Real* nc, Real* mu_c, Real rho, Real* nu, Real* lamc, - Real* cdist, Real* cdist1); - -void get_rain_dsd2_c(Real qr, Real* nr, Real* mu_r, Real* lamr, Real* cdistr, Real* logn0r); - -void calc_rime_density_c(Real T_atm, Real rhofaci, Real table_val_qi_fallspd, Real acn, - Real lamc, Real mu_c, Real qc_incld, Real qc2qi_collect_tend, - Real* vtrmi1, Real* rho_qm_cloud); - -void cldliq_immersion_freezing_c(Real T_atm, Real lamc, Real mu_c, Real cdist1, - Real qc_incld, Real inv_qc_relvar, Real* qc2qi_hetero_freeze_tend, Real* nc2ni_immers_freeze_tend); - -void rain_immersion_freezing_c(Real T_atm, Real lamr, Real mu_r, Real cdistr, - Real qr_incld, Real* qr2qi_immers_freeze_tend, Real* nr2ni_immers_freeze_tend); - -void droplet_self_collection_c(Real rho, Real inv_rho, Real qc_incld, Real mu_c, - Real nu, Real nc2nr_autoconv_tend, Real* nc_accret_tend); - -void cloud_rain_accretion_c(Real rho, Real inv_rho, Real qc_incld, Real nc_incld, - Real qr_incld, Real inv_qc_relvar, Real* qc2qr_accret_tend, Real* nc_accret_tend); - -void cloud_water_autoconversion_c(Real rho, Real qc_incld, Real nc_incld, Real inv_qc_relvar, Real* qc2qr_autoconv_tend, Real* nc2nr_autoconv_tend, Real* ncautr); - -void rain_self_collection_c(Real rho, Real qr_incld, Real nr_incld, Real* nr_selfcollect_tend); - -void impose_max_total_ni_c(Real* ni_local, Real max_total_ni, Real inv_rho_local); - -void ice_melting_c(Real rho,Real T_atm,Real pres,Real rhofaci,Real table_val_qi2qr_melting,Real table_val_qi2qr_vent_melt, - Real latent_heat_vapor,Real latent_heat_fusion,Real dv,Real sc,Real mu,Real kap,Real qv,Real qi_incld, - Real ni_incld,Real* qi2qr_melt_tend,Real* ni2nr_melt_tend); - -void calc_first_order_upwind_step_c(Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, - Real* inv_rho, Real* inv_dz, Int num_arrays, Real** fluxes, Real** vs, Real** qnx); - -void generalized_sedimentation_c(Int kts, Int kte, Int kdir, Int k_qxtop, Int* k_qxbot, Int kbot, Real Co_max, - Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, - Int num_arrays, Real** vs, Real** fluxes, Real** qnx); -void cloud_sedimentation_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* qc_incld, Real* rho, Real* inv_rho, Real* cld_frac_l, Real* acn, Real* inv_dz, - Real dt, Real inv_dt, bool do_predict_nc, - Real* qc, Real* nc, Real* nc_incld, Real* mu_c, Real* lamc, Real* precip_liq_surf, Real* qc_tend, Real* nc_tend); - -void ice_sedimentation_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* rho, Real* inv_rho, Real* rhofaci, Real* cld_frac_i, Real* inv_dz, - Real dt, Real inv_dt, - Real* qi, Real* qi_incld, Real* ni, Real* qm, Real* qm_incld, Real* bm, Real* bm_incld, - Real* ni_incld, Real* precip_ice_surf, Real* qi_tend, Real* ni_tend); - -void rain_sedimentation_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* qr_incld, Real* rho, Real* inv_rho, Real* rhofacr, Real* cld_frac_r, Real* inv_dz, - Real dt, Real inv_dt, - Real* qr, Real* nr, Real* nr_incld, Real* mu_r, Real* lamr, Real* precip_liq_surf, Real* precip_liq_flux, Real* qr_tend, Real* nr_tend); - -void calc_bulk_rho_rime_c(Real qi_tot, Real* qi_rim, Real* bi_rim, Real* rho_rime); - -void homogeneous_freezing_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* T_atm, Real* inv_exner, Real* latent_heat_fusion, - Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm); - -void get_time_space_phys_variables_c(Real T_atm, Real pres, Real rho, Real latent_heat_vapor, Real latent_heat_sublim, Real qv_sat_l, Real qv_sat_i, - Real* mu, Real* dv, Real* sc, Real* dqsdt, Real* dqsidt, Real* ab, Real* abi, Real* kap, Real* eii); - -void update_prognostic_ice_c( - Real qc2qi_hetero_freeze_tend, Real qc2qi_collect_tend, Real qc2qr_ice_shed_tend, Real nc_collect_tend, Real nc2ni_immers_freeze_tend, Real ncshdc, - Real qr2qi_collect_tend, Real nr_collect_tend, Real qr2qi_immers_freeze_tend, Real nr2ni_immers_freeze_tend, Real nr_ice_shed_tend, - Real qi2qr_melt_tend, Real ni2nr_melt_tend, Real qi2qv_sublim_tend, Real qv2qi_vapdep_tend, Real qv2qi_nucleat_tend, Real ni_nucleat_tend, - Real ni_selfcollect_tend, Real ni_sublim_tend, Real qc2qi_berg_tend, Real inv_exner, Real latent_heat_sublim, Real latent_heat_fusion, - bool do_predict_nc, bool log_wetgrowth, Real dt, Real nmltratio, - Real rho_qm_cloud, Real* th_atm, Real* qv, Real* qi, Real* ni, Real* qm, - Real* bm, Real* qc, Real* nc, Real* qr, Real* nr); - -void evaporate_rain_c( Real qr_incld, Real qc_incld, Real nr_incld, Real qi_incld, - Real cld_frac_l, Real cld_frac_r, Real qv, Real qv_prev, - Real qv_sat_l, Real qv_sat_i, Real ab, Real abi, - Real epsr, Real epsi_tot, Real t, Real t_prev, - Real latent_heat_sublim, Real dqsdt, Real dt, - Real* qr2qv_evap_tend, Real* nr_evap_tend); - -void update_prognostic_liquid_c( - Real qc2qr_accret_tend, Real nc_accret_tend, Real qc2qr_autoconv_tend, Real nc2nr_autoconv_tend, Real ncautr, - Real nc_selfcollect_tend, Real qr2qv_evap_tend, Real nr_evap_tend, Real nr_selfcollect_tend , bool do_predict_nc, bool do_prescribed_CCN, - Real inv_rho, Real inv_exner, Real latent_heat_vapor, Real dt, Real* th_atm, Real* qv, - Real* qc, Real* nc, Real* qr, Real* nr); - -void ice_deposition_sublimation_c(Real qi_incld, Real ni_incld, Real t_atm, Real qv_sat_l, Real qv_sat_i, Real epsi, Real abi, Real qv, Real inv_dt, Real* qidep, Real* qi2qv_sublim_tend, Real* ni_sublim_tend, Real* qiberg); - -void compute_rain_fall_velocity_c(Real qr_incld, Real rhofacr, - Real* nr_incld, Real* mu_r, Real* lamr, Real* V_qr, Real* V_nr); - -void ice_cldliq_collection_c(Real rho, Real temp, Real rhofaci, Real table_val_qc2qi_collect, - Real qi_incld,Real qc_incld, Real ni_incld, Real nc_incld, - Real* qc2qi_collect_tend, Real* nc_collect_tend, Real* qc2qr_ice_shed_tend, Real* ncshdc); - -void ice_rain_collection_c(Real rho, Real temp, Real rhofaci, Real logn0r, Real table_val_nr_collect, Real table_val_qr2qi_collect, - Real qi_incld, Real ni_incld, Real qr_incld, Real* qr2qi_collect_tend, Real* nr_collect_tend); - - -void ice_self_collection_c(Real rho, Real rhofaci, Real table_val_ni_self_collect, Real eii, - Real qm_incld, Real qi_incld, Real ni_incld, Real* ni_selfcollect_tend); - -void ice_relaxation_timescale_c(Real rho, Real temp, Real rhofaci, Real table_val_qi2qr_melting, Real table_val_qi2qr_vent_melt, - Real dv, Real mu, Real sc, Real qi_incld, Real ni_incld, - Real* epsi, Real* epsi_tot); - -void calc_liq_relaxation_timescale_c(Real rho, Real f1r, Real f2r, Real dv, - Real mu, Real sc, Real mu_r, Real lamr, - Real cdistr, Real cdist, Real qr_incld, - Real qc_incld, Real* epsr, Real* epsc); - -void ice_nucleation_c(Real temp, Real inv_rho, Real ni, Real ni_activated, - Real qv_supersat_i, Real inv_dt, bool do_predict_nc, bool do_prescribed_CCN, - Real* qv2qi_nucleat_tend, Real* ni_nucleat_tend); - -void ice_cldliq_wet_growth_c(Real rho, Real temp, Real pres, Real rhofaci, Real table_val_qi2qr_melting, - Real table_val_qi2qr_vent_melt, Real latent_heat_vapor, Real latent_heat_fusion, Real dv, - Real kap, Real mu, Real sc, Real qv, Real qc_incld, - Real qi_incld, Real ni_incld, Real qr_incld, bool* log_wetgrowth, - Real* qr2qi_collect_tend, Real* qc2qi_collect_tend, Real* qc_growth_rate, Real* nr_ice_shed_tend, Real* qc2qr_ice_shed_tend); - -Real subgrid_variance_scaling_c(Real relvar, Real expon); - -void check_values_c(Real* qv, Real* temp, Int kts, Int kte, Int timestepcount, - Int force_abort, Int source_ind, Real* col_loc); - -void calculate_incloud_mixingratios_c(Real qc, Real qr, Real qi, Real qm, Real nc, Real nr, Real ni, Real bm, - Real inv_cld_frac_l, Real inv_cld_frac_i, Real inv_cld_frac_r, - Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, - Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld); - -void p3_main_part1_c( - Int kts, Int kte, Int kbot, Int ktop, Int kdir, - bool do_predict_nc, bool do_prescribed_CCN, - Real dt, - Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, - Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, - Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, - Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, - bool* is_nucleat_possible, bool* is_hydromet_present); - -void p3_main_part2_c( - Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, - Real* pres, Real* inv_exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, - Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofaci, Real* acn, - Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, - Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, Real* qc_incld, - Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, - Real* ni_incld, Real* bm_incld, Real* mu_c, Real* nu, Real* lamc, Real* cdist, Real* cdist1, - Real* cdistr, Real* mu_r, Real* lamr, Real* logn0r, Real* qv2qi_depos_tend, Real* precip_total_tend, - Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, - Real* prctot, bool* is_hydromet_present); - -void p3_main_part3_c( - Int kts, Int kte, Int kbot, Int ktop, Int kdir, - Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, - Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, - Real* qi, Real* ni, Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, - Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, - Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); - -void p3_main_c( - Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, - Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, - Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, - Real* precip_ice_surf, Int its, Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, - Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, Real* rho_qi, bool do_predict_nc, bool do_prescribed, Real* dpres, Real* inv_exner, - Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, - Real* liq_ice_exchange, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* qv_prev, Real* t_prev, Real* elapsed_s); - -void ice_supersat_conservation_c(Real* qidep, Real* qinuc, Real cld_frac_i, Real qv, Real qv_sat_i, Real latent_heat_sublim, Real t_atm, Real dt, Real qi2qv_sublim_tend, Real qr2qv_evap_tend); -void nc_conservation_c(Real nc, Real nc_selfcollect_tend, Real dt, Real* nc_collect_tend, Real* nc2ni_immers_freeze_tend, Real* nc_accret_tend, Real* nc2nr_autoconv_tend); -void nr_conservation_c(Real nr, Real ni2nr_melt_tend, Real nr_ice_shed_tend, Real ncshdc, Real nc2nr_autoconv_tend, Real dt, Real nmltratio, Real* nr_collect_tend, Real* nr2ni_immers_freeze_tend, Real* nr_selfcollect_tend, Real* nr_evap_tend); -void ni_conservation_c(Real ni, Real ni_nucleat_tend, Real nr2ni_immers_freeze_tend, Real nc2ni_immers_freeze_tend, Real dt, Real* ni2nr_melt_tend, Real* ni_sublim_tend, Real* ni_selfcollect_tend); -void prevent_liq_supersaturation_c(Real pres, Real t_atm, Real qv, Real latent_heat_vapor, Real latent_heat_sublim, Real dt, Real qidep, Real qinuc, Real* qi2qv_sublim_tend, Real* qr2qv_evap_tend); } // extern "C" : end _c decls namespace scream { namespace p3 { -// -// In all C++ -> Fortran bridge functions you should see p3_init(). P3 needs -// to be initialized since most of its function depend on global tables to be -// populated. The 'true' argument is to set p3 to use its fortran implementations -// instead of calling back to C++. We want this behavior since it doesn't make much -// sense for C++ to bridge over to fortran only to have fortran bridge back to C++. -// If the client wanted the C++ implementation, they should just call it directly. -// - void p3_init_a(P3InitAFortranData& d) { p3_init(); // need to initialize p3 first so that tables are loaded p3_init_a_c(d.ice_table_vals.data(), d.collect_table_vals.data()); } -void find_lookuptable_indices_1a(LookupIceData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - find_lookuptable_indices_1a_c(&d.dumi, &d.dumjj, &d.dumii, &d.dumzz, - &d.dum1, &d.dum4, &d.dum5, &d.dum6, - d.qi, d.ni, d.qm, d.rhop); -} - -void find_lookuptable_indices_1b(LookupIceDataB& d) -{ - p3_init(); - find_lookuptable_indices_1b_c(&d.dumj, &d.dum3, d.qr, d.nr); -} - -void access_lookup_table(AccessLookupTableData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - access_lookup_table_c(d.lid.dumjj, d.lid.dumii, d.lid.dumi, d.index, - d.lid.dum1, d.lid.dum4, d.lid.dum5, &d.proc); -} - -void access_lookup_table_coll(AccessLookupTableCollData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - access_lookup_table_coll_c(d.lid.dumjj, d.lid.dumii, d.lidb.dumj, d.lid.dumi, d.index, - d.lid.dum1, d.lidb.dum3, d.lid.dum4, d.lid.dum5, &d.proc); -} - void BackToCellAverageData::randomize(std::mt19937_64& engine) { // Populate the struct with numbers between 0 and 1. @@ -320,141 +67,6 @@ void BackToCellAverageData::randomize(std::mt19937_64& engine) qc2qi_berg_tend = data_dist(engine); } -void back_to_cell_average(BackToCellAverageData& d) -{ - p3_init(); - back_to_cell_average_c(d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, &d.qc2qr_accret_tend, &d.qr2qv_evap_tend, - &d.qc2qr_autoconv_tend, &d.nc_accret_tend, &d.nc_selfcollect_tend, &d.nc2nr_autoconv_tend, &d.nr_selfcollect_tend, &d.nr_evap_tend, &d.ncautr, - &d.qi2qv_sublim_tend, &d.nr_ice_shed_tend, &d.qc2qi_hetero_freeze_tend, &d.qr2qi_collect_tend, &d.qc2qr_ice_shed_tend, - &d.qi2qr_melt_tend, &d.qc2qi_collect_tend, &d.qr2qi_immers_freeze_tend, &d.ni2nr_melt_tend, &d.nc_collect_tend, &d.ncshdc, &d.nc2ni_immers_freeze_tend, - &d.nr_collect_tend, &d.ni_selfcollect_tend, &d.qv2qi_vapdep_tend, &d.nr2ni_immers_freeze_tend, &d.ni_sublim_tend, &d.qv2qi_nucleat_tend, &d.ni_nucleat_tend, - &d.qc2qi_berg_tend); -} - -void calc_rime_density(CalcRimeDensityData& d) -{ - p3_init(); - calc_rime_density_c(d.T_atm, d.rhofaci, d.table_val_qi_fallspd, d.acn, d.lamc, d.mu_c, - d.qc_incld, d.qc2qi_collect_tend, &d.vtrmi1, &d.rho_qm_cloud); -} - -void cldliq_immersion_freezing(CldliqImmersionFreezingData& d) -{ - p3_init(); - cldliq_immersion_freezing_c(d.T_atm, d.lamc, d.mu_c, d.cdist1, d.qc_incld, d.inv_qc_relvar, - &d.qc2qi_hetero_freeze_tend, &d.nc2ni_immers_freeze_tend); -} - -void droplet_self_collection(DropletSelfCollectionData& d) -{ - p3_init(); - droplet_self_collection_c(d.rho, d.inv_rho, d.qc_incld, d.mu_c, d.nu, d.nc2nr_autoconv_tend, - &d.nc_selfcollect_tend); -} - -void rain_immersion_freezing(RainImmersionFreezingData& d) -{ - p3_init(); - rain_immersion_freezing_c(d.T_atm, d.lamr, d.mu_r, d.cdistr, d.qr_incld, - &d.qr2qi_immers_freeze_tend, &d.nr2ni_immers_freeze_tend); -} - -void cloud_rain_accretion(CloudRainAccretionData& d) -{ - p3_init(); - cloud_rain_accretion_c(d.rho, d.inv_rho, d.qc_incld, d.nc_incld, d.qr_incld, d.inv_qc_relvar, - &d.qc2qr_accret_tend, &d.nc_accret_tend); -} - -void cloud_water_conservation(CloudWaterConservationData& d){ - p3_init(); - cloud_water_conservation_c(d.qc, d.dt, &d.qc2qr_autoconv_tend, &d.qc2qr_accret_tend, &d.qc2qi_collect_tend, &d.qc2qi_hetero_freeze_tend, - &d.qc2qr_ice_shed_tend, &d.qc2qi_berg_tend, &d.qi2qv_sublim_tend, &d.qv2qi_vapdep_tend); -} - -void rain_water_conservation(RainWaterConservationData& d){ - p3_init(); - rain_water_conservation_c(d.qr, d.qc2qr_autoconv_tend, d.qc2qr_accret_tend, d.qi2qr_melt_tend, d.qc2qr_ice_shed_tend, - d.dt, &d.qr2qv_evap_tend, &d.qr2qi_collect_tend, &d.qr2qi_immers_freeze_tend); -} - -void ice_water_conservation(IceWaterConservationData& d){ - p3_init(); - ice_water_conservation_c(d.qi, d.qv2qi_vapdep_tend, d.qv2qi_nucleat_tend, d.qc2qi_berg_tend, d.qr2qi_collect_tend, d.qc2qi_collect_tend, d.qr2qi_immers_freeze_tend, - d.qc2qi_hetero_freeze_tend, d.dt, &d.qi2qv_sublim_tend, &d.qi2qr_melt_tend); -} - -void cloud_water_autoconversion(CloudWaterAutoconversionData& d){ - p3_init(); - cloud_water_autoconversion_c(d.rho, d.qc_incld, d.nc_incld, d.inv_qc_relvar, - &d.qc2qr_autoconv_tend, &d.nc2nr_autoconv_tend, &d.ncautr); -} - -void rain_self_collection(RainSelfCollectionData& d){ - p3_init(); - rain_self_collection_c(d.rho, d.qr_incld, d.nr_incld, &d.nr_selfcollect_tend); -} - -void impose_max_total_ni(ImposeMaxTotalNiData& d){ - p3_init(); - impose_max_total_ni_c(&d.ni_local, d.max_total_ni, d.inv_rho_local); -} - -void get_cloud_dsd2(GetCloudDsd2Data& d) -{ - p3_init(); - Real nc_in = d.nc_in; - get_cloud_dsd2_c(d.qc, &nc_in, &d.mu_c, d.rho, &d.nu, &d.lamc, &d.cdist, &d.cdist1); - d.nc_out = nc_in; -} - -void get_rain_dsd2(GetRainDsd2Data& d) -{ - p3_init(); - Real nr_in = d.nr_in; - get_rain_dsd2_c(d.qr, &nr_in, &d.mu_r, &d.lamr, &d.cdistr, &d.logn0r); - d.nr_out = nr_in; -} - -void ice_cldliq_collection(IceCldliqCollectionData& d) -{ - p3_init(); - ice_cldliq_collection_c(d.rho, d.temp, d.rhofaci, d.table_val_qc2qi_collect, - d.qi_incld, d.qc_incld, d.ni_incld, d.nc_incld, - &d.qc2qi_collect_tend, &d.nc_collect_tend, &d.qc2qr_ice_shed_tend, &d.ncshdc); -} - -void ice_rain_collection(IceRainCollectionData& d) -{ - p3_init(); - ice_rain_collection_c(d.rho, d.temp, d.rhofaci, d.logn0r, d.table_val_nr_collect, d.table_val_qr2qi_collect, - d.qi_incld, d.ni_incld, d.qr_incld, - &d.qr2qi_collect_tend, &d.nr_collect_tend); -} - -void ice_self_collection(IceSelfCollectionData& d) -{ - p3_init(); - ice_self_collection_c(d.rho, d.rhofaci, d.table_val_ni_self_collect, d.eii, d.qm_incld, - d.qi_incld, d.ni_incld, - &d.ni_selfcollect_tend); -} - -void get_time_space_phys_variables(GetTimeSpacePhysVarsData& d) -{ - p3_init(); - get_time_space_phys_variables_c(d.T_atm, d.pres, d.rho, d.latent_heat_vapor, d.latent_heat_sublim, d.qv_sat_l, d.qv_sat_i, &d.mu, &d.dv, - &d.sc, &d.dqsdt, &d.dqsidt, &d.ab, &d.abi, &d.kap, &d.eii); -} - -void ice_relaxation_timescale(IceRelaxationData& d) -{ - p3_init(); - ice_relaxation_timescale_c(d.rho, d.temp, d.rhofaci, d.table_val_qi2qr_melting, d.table_val_qi2qr_vent_melt, - d.dv, d.mu, d.sc, d.qi_incld, d.ni_incld, - &d.epsi, &d.epsi_tot); -} - void CalcLiqRelaxationData::randomize(std::mt19937_64& engine) { // Populate the struct's input fields with numbers between 0 and 1. @@ -473,31 +85,6 @@ void CalcLiqRelaxationData::randomize(std::mt19937_64& engine) qc_incld = data_dist(engine); } -void calc_liq_relaxation_timescale(CalcLiqRelaxationData& d) -{ - p3_init(); - calc_liq_relaxation_timescale_c(d.rho, d.f1r, d.f2r, d.dv, d.mu, d.sc, d.mu_r, - d.lamr, d.cdistr, d.cdist, d.qr_incld, d.qc_incld, &d.epsr, &d.epsc); -} - -void ice_nucleation(IceNucleationData& d) -{ - p3_init(); - ice_nucleation_c(d.temp, d.inv_rho, d.ni, d.ni_activated, - d.qv_supersat_i, d.inv_dt, d.do_predict_nc, d.do_prescribed_CCN, &d.qv2qi_nucleat_tend, &d.ni_nucleat_tend); -} - -void ice_cldliq_wet_growth(IceWetGrowthData& d) -{ - p3_init(); - - ice_cldliq_wet_growth_c(d.rho, d.temp, d.pres, d.rhofaci, d.table_val_qi2qr_melting, - d.table_val_qi2qr_vent_melt, d.latent_heat_vapor, d.latent_heat_fusion, d.dv, - d.kap, d.mu, d.sc, d.qv, d.qc_incld, - d.qi_incld, d.ni_incld, d.qr_incld, &d.log_wetgrowth, - &d.qr2qi_collect_tend, &d.qc2qi_collect_tend, &d.qc_growth_rate, &d.nr_ice_shed_tend, &d.qc2qr_ice_shed_tend); -} - CheckValuesData::CheckValuesData( Int kts_, Int kte_, Int timestepcount_, Int source_ind_, bool force_abort_) : PhysicsTestData( { {(kte_-kts_)+1} }, @@ -507,57 +94,6 @@ CheckValuesData::CheckValuesData( EKAT_REQUIRE_MSG(nk() >= 3 || (kte == 1 && kts == 1), "nk too small to use for col_loc"); } -void check_values(CheckValuesData& d) -{ - p3_init(); - check_values_c(d.qv, d.temp, d.kts, d.kte, d.timestepcount, - d.force_abort, d.source_ind, d.col_loc); -} - -void calculate_incloud_mixingratios(IncloudMixingData& d) -{ - p3_init(); - - calculate_incloud_mixingratios_c(d.qc, d.qr, d.qi, d.qm, d.nc, d.nr, d.ni, d.bm, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, - &d.qc_incld, &d.qr_incld, &d.qi_incld, &d.qm_incld, - &d.nc_incld, &d.nr_incld, &d.ni_incld, &d.bm_incld); - -} - -void update_prognostic_ice(P3UpdatePrognosticIceData& d){ - p3_init(); - update_prognostic_ice_c(d.qc2qi_hetero_freeze_tend, d.qc2qi_collect_tend, d.qc2qr_ice_shed_tend, d.nc_collect_tend, d.nc2ni_immers_freeze_tend, d.ncshdc, - d.qr2qi_collect_tend, d.nr_collect_tend, d.qr2qi_immers_freeze_tend, d.nr2ni_immers_freeze_tend, d.nr_ice_shed_tend, - d.qi2qr_melt_tend, d.ni2nr_melt_tend, d.qi2qv_sublim_tend, d.qv2qi_vapdep_tend, d.qv2qi_nucleat_tend, d.ni_nucleat_tend, - d.ni_selfcollect_tend, d.ni_sublim_tend, d.qc2qi_berg_tend, d.inv_exner, d.latent_heat_sublim, d.latent_heat_fusion, - d.do_predict_nc, d.log_wetgrowth, d.dt, d.nmltratio, - d.rho_qm_cloud, &d.th_atm, &d.qv, &d.qi, &d.ni, &d.qm, - &d.bm, &d.qc, &d.nc, &d.qr, &d.nr); -} - -void evaporate_rain(EvapRainData& d) -{ - p3_init(); - evaporate_rain_c(d.qr_incld,d.qc_incld,d.nr_incld,d.qi_incld, - d.cld_frac_l,d.cld_frac_r,d.qv,d.qv_prev,d.qv_sat_l,d.qv_sat_i, - d.ab,d.abi,d.epsr,d.epsi_tot,d.t,d.t_prev,d.latent_heat_sublim,d.dqsdt,d.dt, - &d.qr2qv_evap_tend,&d.nr_evap_tend); -} - -void update_prognostic_liquid(P3UpdatePrognosticLiqData& d){ - p3_init(); - update_prognostic_liquid_c(d.qc2qr_accret_tend, d.nc_accret_tend, d.qc2qr_autoconv_tend, d.nc2nr_autoconv_tend, d.ncautr, - d.nc_selfcollect_tend, d. qr2qv_evap_tend, d.nr_evap_tend, d.nr_selfcollect_tend , d.do_predict_nc, d.do_prescribed_CCN, - d.inv_rho, d.inv_exner, d.latent_heat_vapor, d.dt, &d.th_atm, &d.qv, - &d.qc, &d.nc, &d.qr, &d.nr); -} - -void ice_deposition_sublimation(IceDepositionSublimationData& d) -{ - p3_init(); - ice_deposition_sublimation_c(d.qi_incld, d.ni_incld, d.T_atm, d.qv_sat_l, d.qv_sat_i, d.epsi, d.abi, d.qv, d.inv_dt, &d.qv2qi_vapdep_tend, &d.qi2qv_sublim_tend, &d.ni_sublim_tend, &d.qc2qi_berg_tend); -} - CalcUpwindData::CalcUpwindData( Int kts_, Int kte_, Int kdir_, Int kbot_, Int k_qxtop_, Int num_arrays_, Real dt_sub_) : PhysicsTestData({ {(kte_ - kts_)+1, num_arrays_}, {(kte_ - kts_)+1} }, @@ -578,15 +114,6 @@ void CalcUpwindData::convert_to_ptr_arr(std::vector& mem_space, Real**& f qnx_ = mem_space.data() + num_arrays*2; } -void calc_first_order_upwind_step(CalcUpwindData& d) -{ - p3_init(); - std::vector tmp; - Real** fluxes, **vs, **qnx; - d.convert_to_ptr_arr(tmp, fluxes, vs, qnx); - calc_first_order_upwind_step_c(d.kts, d.kte, d.kdir, d.kbot, d.k_qxtop, d.dt_sub, d.rho, d.inv_rho, d.inv_dz, d.num_arrays, fluxes, vs, qnx); -} - GenSedData::GenSedData( Int kts_, Int kte_, Int kdir_, Int k_qxtop_, Int k_qxbot_, Int kbot_, Real Co_max_, Real dt_left_, Real prt_accum_, Int num_arrays_) : @@ -594,17 +121,6 @@ GenSedData::GenSedData( Co_max(Co_max_), k_qxbot(k_qxbot_), dt_left(dt_left_), prt_accum(prt_accum_) { } -void generalized_sedimentation(GenSedData& d) -{ - p3_init(); - std::vector tmp; - Real** fluxes, **vs, **qnx; - d.convert_to_ptr_arr(tmp, fluxes, vs, qnx); - generalized_sedimentation_c(d.kts, d.kte, d.kdir, d.k_qxtop, &d.k_qxbot, d.kbot, d.Co_max, - &d.dt_left, &d.prt_accum, d.inv_dz, d.inv_rho, d.rho, - d.num_arrays, fluxes, vs, qnx); -} - CloudSedData::CloudSedData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_, Real dt_, Real inv_dt_, bool do_predict_nc_, Real precip_liq_surf_) : @@ -614,15 +130,6 @@ CloudSedData::CloudSedData( dt(dt_), inv_dt(inv_dt_), do_predict_nc(do_predict_nc_), precip_liq_surf(precip_liq_surf_) {} -void cloud_sedimentation(CloudSedData& d) -{ - p3_init(); - cloud_sedimentation_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.qc_incld, d.rho, d.inv_rho, d.cld_frac_l, d.acn, d.inv_dz, - d.dt, d.inv_dt, d.do_predict_nc, - d.qc, d.nc, d.nc_incld, d.mu_c, d.lamc, &d.precip_liq_surf, d.qc_tend, d.nc_tend); -} - IceSedData::IceSedData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_, Real dt_, Real inv_dt_, Real precip_ice_surf_) : @@ -632,15 +139,6 @@ IceSedData::IceSedData( dt(dt_), inv_dt(inv_dt_), precip_ice_surf(precip_ice_surf_) {} -void ice_sedimentation(IceSedData& d) -{ - p3_init(); - ice_sedimentation_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.rho, d.inv_rho, d.rhofaci, d.cld_frac_i, d.inv_dz, d.dt, d.inv_dt, - d.qi, d.qi_incld, d.ni, d.qm, d.qm_incld, d.bm, d.bm_incld, d.ni_incld, - &d.precip_ice_surf, d.qi_tend, d.ni_tend); -} - RainSedData::RainSedData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_, Real dt_, Real inv_dt_, Real precip_liq_surf_) : @@ -650,21 +148,6 @@ RainSedData::RainSedData( dt(dt_), inv_dt(inv_dt_), precip_liq_surf(precip_liq_surf_) {} -void rain_sedimentation(RainSedData& d) -{ - p3_init(); - rain_sedimentation_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.qr_incld, d.rho, d.inv_rho, d.rhofacr, d.cld_frac_r, d.inv_dz, - d.dt, d.inv_dt, - d.qr, d.nr, d.nr_incld, d.mu_r, d.lamr, &d.precip_liq_surf, d.precip_liq_flux, d.qr_tend, d.nr_tend); -} - -void calc_bulk_rho_rime(CalcBulkRhoRimeData& d) -{ - p3_init(); - calc_bulk_rho_rime_c(d.qi_tot, &d.qi_rim, &d.bi_rim, &d.rho_rime); -} - HomogeneousFreezingData::HomogeneousFreezingData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_) : PhysicsTestData( { {(kte_ - kts_) + 1} }, @@ -672,33 +155,6 @@ HomogeneousFreezingData::HomogeneousFreezingData( kts(kts_), kte(kte_), ktop(ktop_), kbot(kbot_), kdir(kdir_) {} -void homogeneous_freezing(HomogeneousFreezingData& d) -{ - p3_init(); - homogeneous_freezing_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.T_atm, d.inv_exner, d.latent_heat_fusion, - d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.th_atm); -} - -void ice_melting(IceMeltingData& d){ - p3_init(); - ice_melting_c(d.rho,d.T_atm,d.pres,d.rhofaci,d.table_val_qi2qr_melting,d.table_val_qi2qr_vent_melt, - d.latent_heat_vapor,d.latent_heat_fusion,d.dv,d.sc,d.mu,d.kap, - d.qv,d.qi_incld,d.ni_incld,&d.qi2qr_melt_tend,&d.ni2nr_melt_tend); -} - -Real subgrid_variance_scaling(SubgridVarianceScalingData& d){ - p3_init(); - return subgrid_variance_scaling_c(d.relvar,d.expon); -} - -void compute_rain_fall_velocity(ComputeRainFallVelocityData& d) -{ - p3_init(); - compute_rain_fall_velocity_c(d.qr_incld, d.rhofacr, - &d.nr_incld, &d.mu_r, &d.lamr, &d.V_qr, &d.V_nr); -} - P3MainPart1Data::P3MainPart1Data( Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, bool, bool) : @@ -711,21 +167,6 @@ P3MainPart1Data::P3MainPart1Data( do_predict_nc(do_predict_nc_), do_prescribed_CCN(do_prescribed_CCN_), dt(dt_) {} -void p3_main_part1(P3MainPart1Data& d) -{ - p3_init(); - p3_main_part1_c( - d.kts, d.kte, d.kbot, d.ktop, d.kdir, - d.do_predict_nc, d.do_prescribed_CCN, - d.dt, - d.pres, d.dpres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.inv_exner, d.exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.latent_heat_vapor, - d.latent_heat_sublim, d.latent_heat_fusion, - d.T_atm, d.rho, d.inv_rho, d.qv_sat_l, d.qv_sat_i, d.qv_supersat_i, d.rhofacr, d.rhofaci, - d.acn, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.qc_incld, d.qr_incld, d.qi_incld, - d.qm_incld, d.nc_incld, d.nr_incld, d.ni_incld, d.bm_incld, - &d.is_nucleat_possible, &d.is_hydromet_present); -} - /////////////////////////////////////////////////////////////////////////////// P3MainPart2Data::P3MainPart2Data( @@ -742,20 +183,6 @@ P3MainPart2Data::P3MainPart2Data( do_predict_nc(do_predict_nc_), do_prescribed_CCN(do_prescribed_CCN_), dt(dt_), inv_dt(1 / dt) {} -void p3_main_part2(P3MainPart2Data& d) -{ - p3_init(); - p3_main_part2_c( - d.kts, d.kte, d.kbot, d.ktop, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, d.inv_dt, - d.pres, d.inv_exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.ni_activated, d.inv_qc_relvar, - d.cld_frac_i, d.cld_frac_l, d.cld_frac_r, d.qv_prev, d.t_prev, - d.T_atm, d.rho, d.inv_rho, d.qv_sat_l, d.qv_sat_i, d.qv_supersat_i, d.rhofaci, d.acn, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, - d.qm, d.bm, d.latent_heat_vapor, d.latent_heat_sublim, d.latent_heat_fusion, d.qc_incld, d.qr_incld, d.qi_incld, d.qm_incld, d.nc_incld, d.nr_incld, - d.ni_incld, d.bm_incld, d.mu_c, d.nu, d.lamc, d.cdist, d.cdist1, d.cdistr, d.mu_r, d.lamr, d.logn0r, d.qv2qi_depos_tend, d.precip_total_tend, - d.nevapr, d.qr_evap_tend, d.vap_liq_exchange, d.vap_ice_exchange, d.liq_ice_exchange, d.pratot, - d.prctot, &d.is_hydromet_present); -} - /////////////////////////////////////////////////////////////////////////////// P3MainPart3Data::P3MainPart3Data( @@ -771,17 +198,6 @@ P3MainPart3Data::P3MainPart3Data( kts(kts_), kte(kte_), kbot(kbot_), ktop(ktop_), kdir(kdir_) {} -void p3_main_part3(P3MainPart3Data& d) -{ - p3_init(); - p3_main_part3_c( - d.kts, d.kte, d.kbot, d.ktop, d.kdir, - d.inv_exner, d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, - d.rho, d.inv_rho, d.rhofaci, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.latent_heat_vapor, d.latent_heat_sublim, - d.mu_c, d.nu, d.lamc, d.mu_r, d.lamr, d.vap_liq_exchange, - d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc, d.diag_eff_radius_qr); -} - /////////////////////////////////////////////////////////////////////////////// P3MainData::P3MainData( @@ -796,51 +212,6 @@ P3MainData::P3MainData( its(its_), ite(ite_), kts(kts_), kte(kte_), it(it_), dt(dt_), do_predict_nc(do_predict_nc_), do_prescribed_CCN(do_prescribed_CCN_) {} -//This is the variable ordering from micro_p3.F90 -void p3_main(P3MainData& d) -{ - p3_init(); - d.transpose(); - p3_main_c( - d.qc, d.nc, d.qr, d.nr, d.th_atm, d.qv, d.dt, d.qi, d.qm, d.ni, - d.bm, d.pres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.ni_activated, d.inv_qc_relvar, d.it, d.precip_liq_surf, - d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, d.diag_eff_radius_qr, - d.rho_qi, d.do_predict_nc, d.do_prescribed_CCN, d.dpres, d.inv_exner, d.qv2qi_depos_tend, - d.precip_liq_flux, d.precip_ice_flux, d.cld_frac_r, d.cld_frac_l, d.cld_frac_i, - d.liq_ice_exchange, d.vap_liq_exchange, d.vap_ice_exchange, d.qv_prev, d.t_prev, &d.elapsed_s); - d.transpose(); -} - -void ice_supersat_conservation(IceSupersatConservationData& d) -{ - p3_init(); - ice_supersat_conservation_c(&d.qidep, &d.qinuc, d.cld_frac_i, d.qv, d.qv_sat_i, d.latent_heat_sublim, d.t_atm, d.dt, d.qi2qv_sublim_tend, d.qr2qv_evap_tend); -} - -void nc_conservation(NcConservationData& d) -{ - p3_init(); - nc_conservation_c(d.nc, d.nc_selfcollect_tend, d.dt, &d.nc_collect_tend, &d.nc2ni_immers_freeze_tend, &d.nc_accret_tend, &d.nc2nr_autoconv_tend); -} - -void nr_conservation(NrConservationData& d) -{ - p3_init(); - nr_conservation_c(d.nr, d.ni2nr_melt_tend, d.nr_ice_shed_tend, d.ncshdc, d.nc2nr_autoconv_tend, d.dt, d.nmltratio, &d.nr_collect_tend, &d.nr2ni_immers_freeze_tend, &d.nr_selfcollect_tend, &d.nr_evap_tend); -} - -void ni_conservation(NiConservationData& d) -{ - p3_init(); - ni_conservation_c(d.ni, d.ni_nucleat_tend, d.nr2ni_immers_freeze_tend, d.nc2ni_immers_freeze_tend, d.dt, &d.ni2nr_melt_tend, &d.ni_sublim_tend, &d.ni_selfcollect_tend); -} - -void prevent_liq_supersaturation(PreventLiqSupersaturationData& d) -{ - p3_init(); - prevent_liq_supersaturation_c(d.pres, d.t_atm, d.qv, d.latent_heat_vapor, d.latent_heat_sublim, d.dt, d.qidep, d.qinuc, &d.qi2qv_sublim_tend, &d.qr2qv_evap_tend); -} - void IceSupersatConservationData::randomize(std::mt19937_64& engine) { std::uniform_real_distribution data_dist(0.0, 1.0); @@ -954,8 +325,6 @@ void PreventLiqSupersaturationData::randomize(std::mt19937_64& engine) */ } -// end _c impls - /////////////////////////////////////////////////////////////////////////////// std::shared_ptr P3GlobalForFortran::s_views; @@ -977,7 +346,7 @@ void P3GlobalForFortran::deinit() } // -// _f function definitions +// _host function definitions // template @@ -990,7 +359,7 @@ std::vector ptr_to_arr(T** data, int n) } template -void calc_first_order_upwind_step_f_impl( +void calc_first_order_upwind_step_host_impl( Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, Real* inv_rho, Real* inv_dz, Real** fluxes, Real** vs, Real** qnx) @@ -1054,7 +423,7 @@ void calc_first_order_upwind_step_f_impl( } template -void generalized_sedimentation_f_impl( +void generalized_sedimentation_host_impl( Int kts, Int kte, Int kdir, Int k_qxtop, Int* k_qxbot, Int kbot, Real Co_max, Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, Real** vs, Real** fluxes, Real** qnx) @@ -1143,40 +512,40 @@ void generalized_sedimentation_f_impl( *k_qxbot = scalars[2] + 1; } -void calc_first_order_upwind_step_f( +void calc_first_order_upwind_step_host( Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, Real* inv_rho, Real* inv_dz, Int num_arrays, Real** fluxes, Real** vs, Real** qnx) { if (num_arrays == 1) { - calc_first_order_upwind_step_f_impl<1>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); + calc_first_order_upwind_step_host_impl<1>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); } else if (num_arrays == 2) { - calc_first_order_upwind_step_f_impl<2>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); + calc_first_order_upwind_step_host_impl<2>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); } else if (num_arrays == 4) { - calc_first_order_upwind_step_f_impl<4>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); + calc_first_order_upwind_step_host_impl<4>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); } else { EKAT_REQUIRE_MSG(false, "Unsupported num arrays in bridge calc_first_order_upwind_step_f: " << num_arrays); } } -void generalized_sedimentation_f( +void generalized_sedimentation_host( Int kts, Int kte, Int kdir, Int k_qxtop, Int* k_qxbot, Int kbot, Real Co_max, Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, Int num_arrays, Real** vs, Real** fluxes, Real** qnx) { if (num_arrays == 1) { - generalized_sedimentation_f_impl<1>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, + generalized_sedimentation_host_impl<1>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, vs, fluxes, qnx); } else if (num_arrays == 2) { - generalized_sedimentation_f_impl<2>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, + generalized_sedimentation_host_impl<2>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, vs, fluxes, qnx); } else if (num_arrays == 4) { - generalized_sedimentation_f_impl<4>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, + generalized_sedimentation_host_impl<4>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, vs, fluxes, qnx); } else { @@ -1184,7 +553,7 @@ void generalized_sedimentation_f( } } -void cloud_sedimentation_f( +void cloud_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qc_incld, Real* rho, Real* inv_rho, Real* cld_frac_l, Real* acn, Real* inv_dz, Real dt, Real inv_dt, bool do_predict_nc, @@ -1251,7 +620,7 @@ void cloud_sedimentation_f( ekat::device_to_host({qc, nc, nc_incld, mu_c, lamc, qc_tend, nc_tend}, nk, inout_views); } -void ice_sedimentation_f( +void ice_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* rho, Real* inv_rho, Real* rhofaci, Real* cld_frac_i, Real* inv_dz, Real dt, Real inv_dt, @@ -1324,7 +693,7 @@ void ice_sedimentation_f( ekat::device_to_host({qi, qi_incld, ni, ni_incld, qm, qm_incld, bm, bm_incld, qi_tend, ni_tend}, nk, inout_views); } -void rain_sedimentation_f( +void rain_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qr_incld, Real* rho, Real* inv_rho, Real* rhofacr, Real* cld_frac_r, Real* inv_dz, Real dt, Real inv_dt, @@ -1399,7 +768,7 @@ void rain_sedimentation_f( ekat::device_to_host({qr, nr, nr_incld, mu_r, lamr, qr_tend, nr_tend, precip_liq_flux}, sizes_out, inout_views); } -void homogeneous_freezing_f( +void homogeneous_freezing_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm) @@ -1460,7 +829,7 @@ void homogeneous_freezing_f( ekat::device_to_host({qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, nk, inout_views); } -void check_values_f(Real* qv, Real* temp, Int kstart, Int kend, +void check_values_host(Real* qv, Real* temp, Int kstart, Int kend, Int timestepcount, bool force_abort, Int source_ind, Real* col_loc) { using P3F = Functions; @@ -1493,7 +862,7 @@ void check_values_f(Real* qv, Real* temp, Int kstart, Int kend, }); } -void p3_main_part1_f( +void p3_main_part1_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, @@ -1605,7 +974,7 @@ void p3_main_part1_f( *is_hydromet_present = bools_h(1); } -void p3_main_part2_f( +void p3_main_part2_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, @@ -1761,7 +1130,7 @@ void p3_main_part2_f( *is_hydromet_present = bools_h(0); } -void p3_main_part3_f( +void p3_main_part3_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, @@ -1869,7 +1238,7 @@ void p3_main_part3_f( nk, inout_views); } -Int p3_main_f( +Int p3_main_host( Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 881356f3d9b..7042f0f9fb8 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -10,18 +10,34 @@ #include #include // for shared_ptr -// -// Bridge functions to call fortran version of p3 functions from C++ -// - namespace scream { namespace p3 { -// +/////////////////////////////////////////////////////////////////////////////// + +struct P3InitAFortranData +{ + // Must use Host as device, f90 code might not be able to use Device memory + using P3F = Functions; + using P3C = typename P3F::P3C; + + using view_ice_table = typename P3F::KT::template lview; + using view_collect_table = typename P3F::KT::template lview; + + // Need to be LayoutLeft to be fortran compatible + view_ice_table ice_table_vals; + view_collect_table collect_table_vals; + + P3InitAFortranData() : + ice_table_vals("P3InitAFortranData::ice_table_vals"), + collect_table_vals("P3InitAFortranData::collect_table_vals") + {} +}; + +/////////////////////////////////////////////////////////////////////////////// + // Singleton for holding the same global data that are maintained in -// micro_p3, but for use in C++. This data is necessary to complete -// the "bridge" when calling C++ from micro_p3. -// +// micro_p3, but for use in C++. struct P3GlobalForFortran { using P3F = Functions; @@ -64,26 +80,10 @@ struct P3GlobalForFortran /////////////////////////////////////////////////////////////////////////////// -struct P3InitAFortranData -{ - // Must use Host as device, f90 code might not be able to use Device memory - using P3F = Functions; - using P3C = typename P3F::P3C; - - using view_ice_table = typename P3F::KT::template lview; - using view_collect_table = typename P3F::KT::template lview; - - // Need to be LayoutLeft to be fortran compatible - view_ice_table ice_table_vals; - view_collect_table collect_table_vals; - - P3InitAFortranData() : - ice_table_vals("P3InitAFortranData::ice_table_vals"), - collect_table_vals("P3InitAFortranData::collect_table_vals") - {} -}; - -/////////////////////////////////////////////////////////////////////////////// +/** + * Structs for holding data related to specific P3 calls; these are used for + * the BFB unit tests. + */ struct LookupIceData { @@ -882,98 +882,50 @@ struct PreventLiqSupersaturationData { PTD_RW_SCALARS_ONLY(2, qi2qv_sublim_tend, qr2qv_evap_tend); }; -// Glue functions to call fortran from from C++ with the Data struct void p3_init_a(P3InitAFortranData& d); -void find_lookuptable_indices_1a(LookupIceData& d); -void find_lookuptable_indices_1b(LookupIceDataB& d); -void access_lookup_table(AccessLookupTableData& d); -void access_lookup_table_coll(AccessLookupTableCollData& d); -void back_to_cell_average(BackToCellAverageData& d); -void cloud_water_conservation(CloudWaterConservationData& d); -void rain_water_conservation(RainWaterConservationData& d); -void ice_water_conservation(IceWaterConservationData& d); -void calc_rime_density(CalcRimeDensityData& d); -void cldliq_immersion_freezing(CldliqImmersionFreezingData& d); -void rain_immersion_freezing(RainImmersionFreezingData& d); -void droplet_self_collection(DropletSelfCollectionData& d); -void cloud_rain_accretion(CloudRainAccretionData& d); -void cloud_water_autoconversion(CloudWaterAutoconversionData& d); -void rain_self_collection(RainSelfCollectionData& d); -void impose_max_total_ni(ImposeMaxTotalNiData& d); -void ice_melting(IceMeltingData& d); -Real subgrid_variance_scaling(SubgridVarianceScalingData& d); -void get_cloud_dsd2(GetCloudDsd2Data& d); -void get_rain_dsd2(GetRainDsd2Data& d); -void calc_first_order_upwind_step(CalcUpwindData& d); -void generalized_sedimentation(GenSedData& d); -void cloud_sedimentation(CloudSedData& d); -void ice_sedimentation(IceSedData& d); -void rain_sedimentation(RainSedData& d); -void calc_bulk_rho_rime(CalcBulkRhoRimeData& d); -void homogeneous_freezing(HomogeneousFreezingData& d); -void compute_rain_fall_velocity(ComputeRainFallVelocityData& d); -void get_time_space_phys_variables(GetTimeSpacePhysVarsData& d); -void update_prognostic_ice(P3UpdatePrognosticIceData& d); -void evaporate_rain(EvapRainData& d); -void update_prognostic_liquid(P3UpdatePrognosticLiqData& d); -void ice_deposition_sublimation(IceDepositionSublimationData& d); -void ice_cldliq_collection(IceCldliqCollectionData& d); -void ice_rain_collection(IceRainCollectionData& d); -void ice_self_collection(IceSelfCollectionData& d); -void ice_relaxation_timescale(IceRelaxationData& d); -void calc_liq_relaxation_timescale(CalcLiqRelaxationData& d); -void ice_nucleation(IceNucleationData& d); -void ice_cldliq_wet_growth(IceWetGrowthData& d); -void check_values(CheckValuesData& d); -void calculate_incloud_mixingratios(IncloudMixingData& d); -void p3_main_part1(P3MainPart1Data& d); -void p3_main_part2(P3MainPart2Data& d); -void p3_main_part3(P3MainPart3Data& d); -void p3_main(P3MainData& d); - -void ice_supersat_conservation(IceSupersatConservationData& d); -void nc_conservation(NcConservationData& d); -void nr_conservation(NrConservationData& d); -void ni_conservation(NiConservationData& d); -void prevent_liq_supersaturation(PreventLiqSupersaturationData& d); -extern "C" { // _f function decls - -void calc_first_order_upwind_step_f( + +/** + * Convenience functions for calling p3 routines from the host with scalar data. + * These function will pack your data, sync it to device, call the p3 function, + * then sync back to host and unpack. These are used by the BFB unit tests. + */ + +void calc_first_order_upwind_step_host( Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, Real* inv_rho, Real* inv_dz, Int num_arrays, Real** fluxes, Real** vs, Real** qnx); -void generalized_sedimentation_f(Int kts, Int kte, Int kdir, Int k_qxtop, Int *k_qxbot, Int kbot, Real Co_max, +void generalized_sedimentation_host(Int kts, Int kte, Int kdir, Int k_qxtop, Int *k_qxbot, Int kbot, Real Co_max, Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, Int num_arrays, Real** vs, Real** fluxes, Real** qnx); -void cloud_sedimentation_f( +void cloud_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qc_incld, Real* rho, Real* inv_rho, Real* cld_frac_l, Real* acn, Real* inv_dz, Real dt, Real inv_dt, bool do_predict_nc, Real* qc, Real* nc, Real* nc_incld, Real* mu_c, Real* lamc, Real* precip_liq_surf, Real* qc_tend, Real* nc_tend); -void ice_sedimentation_f( +void ice_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* rho, Real* inv_rho, Real* rhofaci, Real* cld_frac_i, Real* inv_dz, Real dt, Real inv_dt, Real* qi, Real* qi_incld, Real* ni, Real* qm, Real* qm_incld, Real* bm, Real* bm_incld, Real* ni_incld, Real* precip_ice_surf, Real* qi_tend, Real* ni_tend); -void rain_sedimentation_f( +void rain_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qr_incld, Real* rho, Real* inv_rho, Real* rhofacr, Real* cld_frac_r, Real* inv_dz, Real dt, Real inv_dt, Real* qr, Real* nr, Real* nr_incld, Real* mu_r, Real* lamr, Real* precip_liq_surf, Real* precip_liq_flux, Real* qr_tend, Real* nr_tend); -void homogeneous_freezing_f( +void homogeneous_freezing_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm); -void check_values_f(Real* Qv, Real* temp, Int kstart, Int kend, - Int timestepcount, bool force_abort, Int source_ind, Real* col_loc); +void check_values_host(Real* Qv, Real* temp, Int kstart, Int kend, + Int timestepcount, bool force_abort, Int source_ind, Real* col_loc); -void p3_main_part1_f( +void p3_main_part1_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, @@ -984,7 +936,7 @@ void p3_main_part1_f( Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, bool* is_nucleat_possible, bool* is_hydromet_present); -void p3_main_part2_f( +void p3_main_part2_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, @@ -993,14 +945,14 @@ void p3_main_part2_f( Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, Real* prctot, bool* is_hydromet_present); -void p3_main_part3_f( +void p3_main_part3_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); -Int p3_main_f( +Int p3_main_host( Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, @@ -1009,8 +961,6 @@ Int p3_main_f( Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, Real* liq_ice_exchange, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* qv_prev, Real* t_prev); -} // end _f function decls - } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index ea0a18411c1..34ee99cde33 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -129,59 +129,6 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) end subroutine p3_init_c - subroutine p3_main_c(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & - pres,dz,nc_nuceat_tend,nccn_prescribed,ni_activated,inv_qc_relvar,it,precip_liq_surf,precip_ice_surf,its,ite,kts,kte,diag_eff_radius_qc, & - diag_eff_radius_qi,diag_eff_radius_qr,rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend, & - precip_liq_flux,precip_ice_flux,cld_frac_r,cld_frac_l,cld_frac_i,liq_ice_exchange, & - vap_liq_exchange, vap_ice_exchange, qv_prev, t_prev, elapsed_s) bind(C) - use micro_p3, only : p3_main - - real(kind=c_real), intent(inout), dimension(its:ite,kts:kte) :: qc, nc, qr, nr, qv, th_atm - real(kind=c_real), intent(inout), dimension(its:ite,kts:kte) :: qi, qm, ni, bm - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: pres, dz - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: nc_nuceat_tend,nccn_prescribed,ni_activated - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: inv_qc_relvar - real(kind=c_real), value, intent(in) :: dt - real(kind=c_real), intent(out), dimension(its:ite) :: precip_liq_surf, precip_ice_surf - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qc - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qi - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qr - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: rho_qi - integer(kind=c_int), value, intent(in) :: its,ite, kts,kte, it - logical(kind=c_bool), value, intent(in) :: do_predict_nc,do_prescribed_CCN - - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: dpres - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: inv_exner - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: qv2qi_depos_tend - real(kind=c_real), intent(out), dimension(its:ite,kts:kte+1) :: precip_liq_flux - real(kind=c_real), intent(out), dimension(its:ite,kts:kte+1) :: precip_ice_flux - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: cld_frac_i, cld_frac_l, cld_frac_r - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: liq_ice_exchange - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: vap_liq_exchange - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: vap_ice_exchange - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: qv_prev - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: t_prev - - real(kind=c_real), intent(out) :: elapsed_s - - real(kind=c_real), dimension(its:ite,kts:kte,49) :: p3_tend_out - real(kind=c_real), dimension(its:ite,3) :: col_location - real(kind=c_real), dimension(its:ite,kts:kte) :: mu_c, lamc - real(kind=c_real), dimension(its:ite,kts:kte) :: precip_total_tend - real(kind=c_real), dimension(its:ite,kts:kte) :: nevapr - real(kind=c_real), dimension(its:ite,kts:kte) :: qr_evap_tend - integer :: i - do i = its,ite - col_location(i,:) = real(i) - end do - - call p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & - pres,dz,nc_nuceat_tend,nccn_prescribed,ni_activated,inv_qc_relvar,it,precip_liq_surf,precip_ice_surf,its,ite,kts,kte,diag_eff_radius_qc, & - diag_eff_radius_qi,diag_eff_radius_qr, rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend,precip_total_tend,nevapr, & - qr_evap_tend,precip_liq_flux,precip_ice_flux,cld_frac_r,cld_frac_l,cld_frac_i,p3_tend_out,mu_c,lamc,liq_ice_exchange,& - vap_liq_exchange,vap_ice_exchange,qv_prev,t_prev,col_location,elapsed_s) - end subroutine p3_main_c - subroutine micro_p3_utils_init_c(Cpair, Rair, RH2O, RHO_H2O, & MWH2O, MWdry, gravit, LatVap, LatIce, & CpLiq, Tmelt, Pi, masterproc) bind(C) @@ -217,772 +164,4 @@ subroutine p3_init_a_c(ice_table_vals_c, collect_table_vals_c) bind(C) collect_table_vals_c(:,:,:,:,:) = collect_table_vals(:,:,:,:,:) end subroutine p3_init_a_c - subroutine find_lookuptable_indices_1a_c(dumi,dumjj,dumii,dumzz,dum1,dum4,dum5,dum6, & - qi,ni,qm,rhop) bind(C) - use micro_p3, only: find_lookupTable_indices_1a - use micro_p3_utils, only: densize,rimsize,isize - - ! arguments: - integer(kind=c_int), intent(out) :: dumi,dumjj,dumii,dumzz - real(kind=c_real), intent(out) :: dum1,dum4,dum5,dum6 - real(kind=c_real), value, intent(in) :: qi,ni,qm,rhop - - call find_lookupTable_indices_1a(dumi, dumjj, dumii, dumzz, dum1, dum4, dum5, dum6, & - isize, rimsize, densize, qi, ni, qm, rhop) - end subroutine find_lookuptable_indices_1a_c - - subroutine find_lookuptable_indices_1b_c(dumj,dum3,qr,nr) bind(C) - use micro_p3, only: find_lookupTable_indices_1b - use micro_p3_utils, only: rcollsize - - integer(kind=c_int), intent(out) :: dumj - real(kind=c_real), intent(out) :: dum3 - real(kind=c_real), value, intent(in) :: qr, nr - - call find_lookupTable_indices_1b(dumj, dum3, rcollsize, qr, nr) - end subroutine find_lookupTable_indices_1b_c - - subroutine access_lookup_table_c(dumjj,dumii,dumi,index,dum1,dum4,dum5,proc) bind(C) - use micro_p3, only: access_lookup_table - - integer(kind=c_int), value, intent(in) :: dumjj, dumii, dumi, index - real(kind=c_real), value, intent(in) :: dum1, dum4, dum5 - real(kind=c_real), intent(out) :: proc - - call access_lookup_table(dumjj,dumii,dumi,index,dum1,dum4,dum5,proc) - end subroutine access_lookup_table_c - - subroutine access_lookup_table_coll_c(dumjj,dumii,dumj,dumi,index,dum1,dum3,dum4,dum5,proc) bind(C) - use micro_p3, only: access_lookup_table_coll - - integer(kind=c_int), value, intent(in) :: dumjj,dumii,dumj,dumi,index - real(kind=c_real), value, intent(in) :: dum1,dum3,dum4,dum5 - real(kind=c_real), intent(out) :: proc - - call access_lookup_table_coll(dumjj,dumii,dumj,dumi,index,dum1,dum3,dum4,dum5,proc) - end subroutine access_lookup_table_coll_c - - subroutine back_to_cell_average_c(cld_frac_l,cld_frac_r,cld_frac_i, qc2qr_accret_tend,qr2qv_evap_tend,qc2qr_autoconv_tend,& - nc_accret_tend,nc_selfcollect_tend,nc2nr_autoconv_tend,nr_selfcollect_tend,nr_evap_tend,ncautr,qi2qv_sublim_tend,nr_ice_shed_tend,qc2qi_hetero_freeze_tend,& - qr2qi_collect_tend,qc2qr_ice_shed_tend,qi2qr_melt_tend,qc2qi_collect_tend,qr2qi_immers_freeze_tend,ni2nr_melt_tend,nc_collect_tend,ncshdc,nc2ni_immers_freeze_tend,nr_collect_tend,ni_selfcollect_tend,& - qv2qi_vapdep_tend,nr2ni_immers_freeze_tend,ni_sublim_tend,qv2qi_nucleat_tend,ni_nucleat_tend,qc2qi_berg_tend) bind(C) - - use micro_p3, only: back_to_cell_average - real(kind=c_real), value, intent(in) :: cld_frac_l, cld_frac_r, cld_frac_i - - real(kind=c_real), intent(inout) :: qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend, nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, & - nr_selfcollect_tend, nr_evap_tend, ncautr, qi2qv_sublim_tend, & - nr_ice_shed_tend, qc2qi_hetero_freeze_tend, qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, & - qr2qi_immers_freeze_tend, ni2nr_melt_tend, nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend,& - ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, & - qc2qi_berg_tend - - call back_to_cell_average(cld_frac_l, cld_frac_r, cld_frac_i, qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend,& - nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, nr_selfcollect_tend, nr_evap_tend, ncautr, qi2qv_sublim_tend, nr_ice_shed_tend, qc2qi_hetero_freeze_tend,& - qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, ni2nr_melt_tend, nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend, ni_selfcollect_tend,& - qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, qc2qi_berg_tend) - end subroutine back_to_cell_average_c - - subroutine cloud_water_conservation_c(qc,dt,qc2qr_autoconv_tend,qc2qr_accret_tend,qc2qi_collect_tend,qc2qi_hetero_freeze_tend,qc2qr_ice_shed_tend, & - qc2qi_berg_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend) bind(C) - use micro_p3, only: cloud_water_conservation - - real(kind=c_real), value, intent(in) :: qc, dt - real(kind=c_real), intent(inout) :: qc2qr_autoconv_tend, qc2qr_accret_tend, qc2qi_collect_tend, qc2qi_hetero_freeze_tend, qc2qr_ice_shed_tend, qc2qi_berg_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend - - call cloud_water_conservation(qc,dt,qc2qr_autoconv_tend,qc2qr_accret_tend,qc2qi_collect_tend,qc2qi_hetero_freeze_tend,qc2qr_ice_shed_tend,qc2qi_berg_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend) - end subroutine cloud_water_conservation_c - - subroutine rain_water_conservation_c(qr,qc2qr_autoconv_tend,qc2qr_accret_tend,qi2qr_melt_tend,qc2qr_ice_shed_tend,dt, & - qr2qv_evap_tend,qr2qi_collect_tend,qr2qi_immers_freeze_tend) bind(C) - use micro_p3, only: rain_water_conservation - - real(kind=c_real), value, intent(in) :: qr, qc2qr_autoconv_tend, qc2qr_accret_tend, qi2qr_melt_tend, qc2qr_ice_shed_tend, dt - real(kind=c_real), intent(inout) :: qr2qv_evap_tend, qr2qi_collect_tend, qr2qi_immers_freeze_tend - - call rain_water_conservation(qr,qc2qr_autoconv_tend,qc2qr_accret_tend,qi2qr_melt_tend,qc2qr_ice_shed_tend,dt,qr2qv_evap_tend,qr2qi_collect_tend,qr2qi_immers_freeze_tend) - end subroutine rain_water_conservation_c - - subroutine rain_self_collection_c(rho, qr_incld, nr_incld, nr_selfcollect_tend) bind(C) - use micro_p3, only: rain_self_collection - - real(kind=c_real), value, intent(in) :: rho, qr_incld, nr_incld - real(kind=c_real), intent(out) :: nr_selfcollect_tend - - call rain_self_collection(rho, qr_incld, nr_incld, nr_selfcollect_tend) - end subroutine rain_self_collection_c - - subroutine ice_water_conservation_c(qi,qv2qi_vapdep_tend,qv2qi_nucleat_tend,qc2qi_berg_tend,qr2qi_collect_tend,qc2qi_collect_tend,qr2qi_immers_freeze_tend,qc2qi_hetero_freeze_tend,dt, & - qi2qv_sublim_tend,qi2qr_melt_tend) bind(C) - use micro_p3, only: ice_water_conservation - - real(kind=c_real), value, intent(in) :: qi, qv2qi_vapdep_tend, qv2qi_nucleat_tend, qr2qi_collect_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, qc2qi_hetero_freeze_tend, qc2qi_berg_tend, dt - real(kind=c_real), intent(inout) :: qi2qv_sublim_tend, qi2qr_melt_tend - - call ice_water_conservation(qi,qv2qi_vapdep_tend,qv2qi_nucleat_tend,qr2qi_collect_tend,qc2qi_collect_tend,qr2qi_immers_freeze_tend,qc2qi_hetero_freeze_tend,qc2qi_berg_tend,dt,qi2qv_sublim_tend,qi2qr_melt_tend) - end subroutine ice_water_conservation_c - - subroutine get_cloud_dsd2_c(qc,nc,mu_c,rho,nu,lamc,cdist,cdist1) bind(C) - use micro_p3, only: get_cloud_dsd2 - use micro_p3_utils, only: dnu - - !arguments: - real(kind=c_real), value, intent(in) :: qc,rho - real(kind=c_real), intent(inout) :: nc - real(kind=c_real), intent(out) :: mu_c,nu,lamc,cdist,cdist1 - - call get_cloud_dsd2(qc,nc,mu_c,rho,nu,dnu,lamc,cdist,cdist1) - end subroutine get_cloud_dsd2_c - - subroutine get_rain_dsd2_c(qr,nr,mu_r,lamr,cdistr,logn0r) bind(C) - use micro_p3, only: get_rain_dsd2 - - !arguments: - real(kind=c_real), value, intent(in) :: qr - real(kind=c_real), intent(inout) :: nr - real(kind=c_real), intent(out) :: lamr,mu_r,cdistr,logn0r - - call get_rain_dsd2(qr,nr,mu_r,lamr,cdistr,logn0r) - end subroutine get_rain_dsd2_c - - subroutine calc_rime_density_c(T_atm,rhofaci,table_val_qi_fallspd,acn,lamc,mu_c,qc_incld,qc2qi_collect_tend, & - vtrmi1,rho_qm_cloud) bind(C) - - use micro_p3, only: calc_rime_density - real(kind=c_real), value, intent(in) :: T_atm, rhofaci, table_val_qi_fallspd, acn, lamc, mu_c, qc_incld, qc2qi_collect_tend - real(kind=c_real), intent(out) :: vtrmi1, rho_qm_cloud - - call calc_rime_density(T_atm, rhofaci, table_val_qi_fallspd, acn, lamc, mu_c, qc_incld, qc2qi_collect_tend, vtrmi1, rho_qm_cloud) - end subroutine calc_rime_density_c - - subroutine cldliq_immersion_freezing_c(T_atm,lamc,mu_c,cdist1,qc_incld,inv_qc_relvar,qc2qi_hetero_freeze_tend,nc2ni_immers_freeze_tend) bind(C) - - use micro_p3, only: cldliq_immersion_freezing - real(kind=c_real), value, intent(in) :: T_atm, lamc, mu_c, cdist1, qc_incld,inv_qc_relvar - real(kind=c_real), intent(out) :: qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend - - call cldliq_immersion_freezing(T_atm, lamc, mu_c, cdist1, qc_incld, inv_qc_relvar, qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend) - end subroutine cldliq_immersion_freezing_c - - subroutine rain_immersion_freezing_c(T_atm,lamr,mu_r,cdistr,qr_incld,qr2qi_immers_freeze_tend,nr2ni_immers_freeze_tend) bind(C) - - use micro_p3, only: rain_immersion_freezing - real(kind=c_real), value, intent(in) :: T_atm, lamr, mu_r, cdistr, qr_incld - real(kind=c_real), intent(out) :: qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend - - call rain_immersion_freezing(T_atm, lamr, mu_r, cdistr, qr_incld, qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend) - end subroutine rain_immersion_freezing_c - - subroutine droplet_self_collection_c(rho,inv_rho,qc_incld,mu_c,nu,nc2nr_autoconv_tend,nc_selfcollect_tend) bind(C) - - use micro_p3, only: droplet_self_collection - real(kind=c_real), value, intent(in) :: rho, inv_rho, qc_incld, mu_c, nu, nc2nr_autoconv_tend - real(kind=c_real), intent(out) :: nc_selfcollect_tend - - call droplet_self_collection(rho, inv_rho, qc_incld, mu_c, nu, nc2nr_autoconv_tend, nc_selfcollect_tend) - end subroutine droplet_self_collection_c - - subroutine cloud_rain_accretion_c(rho,inv_rho,qc_incld,nc_incld,qr_incld,inv_qc_relvar,qc2qr_accret_tend,nc_accret_tend) bind(C) - - use micro_p3, only: cloud_rain_accretion - real(kind=c_real), value, intent(in) :: rho, inv_rho, qc_incld, nc_incld, qr_incld,inv_qc_relvar - real(kind=c_real), intent(out) :: qc2qr_accret_tend, nc_accret_tend - - call cloud_rain_accretion(rho, inv_rho, qc_incld, nc_incld, qr_incld, inv_qc_relvar, qc2qr_accret_tend, nc_accret_tend) - end subroutine cloud_rain_accretion_c - - subroutine cloud_water_autoconversion_c(rho,qc_incld,nc_incld,inv_qc_relvar,qc2qr_autoconv_tend,nc2nr_autoconv_tend,ncautr) bind(C) - - use micro_p3, only: cloud_water_autoconversion - real(kind=c_real), value, intent(in) :: rho, qc_incld, nc_incld,inv_qc_relvar - real(kind=c_real), intent(inout) :: qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr - - call cloud_water_autoconversion(rho, qc_incld, nc_incld, inv_qc_relvar, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr) - end subroutine cloud_water_autoconversion_c - - subroutine impose_max_total_ni_c(ni_local, max_total_ni, inv_rho_local) bind(C) - use micro_p3, only: impose_max_total_ni - - real(kind=c_real), intent(inout) :: ni_local - real(kind=c_real), value, intent(in) :: max_total_ni, inv_rho_local - - call impose_max_total_ni(ni_local, max_total_ni, inv_rho_local) - end subroutine impose_max_total_ni_c - - subroutine calc_first_order_upwind_step_c(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, num_arrays, fluxes, vs, qnx) bind(C) - use micro_p3, only: calc_first_order_upwind_step, realptr - - !arguments: - integer(kind=c_int), value, intent(in) :: kts, kte, kdir, kbot, k_qxtop, num_arrays - real(kind=c_real), value, intent(in) :: dt_sub - real(kind=c_real), dimension(kts:kte), intent(in) :: rho, inv_rho, inv_dz - type(c_ptr), intent(in), dimension(num_arrays) :: fluxes, vs, qnx - - type(realptr), dimension(num_arrays) :: fluxes_f, vs_f, qnx_f - integer :: i - - do i = 1, num_arrays - call c_f_pointer(fluxes(i), fluxes_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(vs(i), vs_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(qnx(i), qnx_f(i)%p , [(kte-kts)+1]) - end do - - call calc_first_order_upwind_step(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, num_arrays, fluxes_f, vs_f, qnx_f) - - end subroutine calc_first_order_upwind_step_c - - subroutine generalized_sedimentation_c(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, num_arrays, vs, fluxes, qnx) bind(C) - use micro_p3, only: generalized_sedimentation, realptr - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, kdir, k_qxtop, kbot, num_arrays - integer(kind=c_int), intent(inout) :: k_qxbot - real(kind=c_real), value, intent(in) :: Co_max - real(kind=c_real), intent(inout) :: dt_left, prt_accum - real(kind=c_real), dimension(kts:kte), intent(in) :: inv_dz, inv_rho, rho - type(c_ptr), intent(in), dimension(num_arrays) :: vs, fluxes, qnx - - type(realptr), dimension(num_arrays) :: fluxes_f, vs_f, qnx_f - integer :: i - - do i = 1, num_arrays - call c_f_pointer(fluxes(i), fluxes_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(vs(i), vs_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(qnx(i), qnx_f(i)%p , [(kte-kts)+1]) - end do - - call generalized_sedimentation(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, num_arrays, vs_f, fluxes_f, qnx_f) - - end subroutine generalized_sedimentation_c - - subroutine cloud_sedimentation_c(kts,kte,ktop,kbot,kdir, & - qc_incld,rho,inv_rho,cld_frac_l,acn,inv_dz,& - dt,inv_dt,do_predict_nc, & - qc, nc, nc_incld,mu_c,lamc,precip_liq_surf,qc_tend,nc_tend) bind(C) - use micro_p3, only: cloud_sedimentation, dnu - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - - real(kind=c_real), intent(in), dimension(kts:kte) :: rho - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_rho - real(kind=c_real), intent(in), dimension(kts:kte) :: cld_frac_l - real(kind=c_real), intent(in), dimension(kts:kte) :: acn - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_dz - - real(kind=c_real), value, intent(in) :: dt - real(kind=c_real), value, intent(in) :: inv_dt - logical(kind=c_bool), value, intent(in) :: do_predict_nc - - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: mu_c - real(kind=c_real), intent(inout), dimension(kts:kte) :: lamc - real(kind=c_real), intent(inout) :: precip_liq_surf - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc_tend - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc_tend - - call cloud_sedimentation(kts,kte,ktop,kbot,kdir, & - qc_incld,rho,inv_rho,cld_frac_l,acn,inv_dz,& - dt,inv_dt,dnu,do_predict_nc, & - qc, nc, nc_incld,mu_c,lamc,precip_liq_surf,qc_tend,nc_tend) - - end subroutine cloud_sedimentation_c - - subroutine ice_sedimentation_c(kts,kte,ktop,kbot,kdir, & - rho,inv_rho,rhofaci,cld_frac_i,inv_dz,dt,inv_dt, & - qi,qi_incld,ni,qm,qm_incld,bm,bm_incld,ni_incld,precip_ice_surf,qi_tend,ni_tend) bind(C) - use micro_p3, only: ice_sedimentation - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - - real(kind=c_real), intent(in), dimension(kts:kte) :: rho - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_rho - real(kind=c_real), intent(in), dimension(kts:kte) :: rhofaci - real(kind=c_real), intent(in), dimension(kts:kte) :: cld_frac_i - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_dz - real(kind=c_real), value, intent(in) :: dt, inv_dt - - real(kind=c_real), intent(inout), dimension(kts:kte), target :: qi - real(kind=c_real), intent(inout), dimension(kts:kte) :: qi_incld - real(kind=c_real), intent(inout), dimension(kts:kte), target :: ni - real(kind=c_real), intent(inout), dimension(kts:kte) :: ni_incld - real(kind=c_real), intent(inout), dimension(kts:kte), target :: qm - real(kind=c_real), intent(inout), dimension(kts:kte) :: qm_incld - real(kind=c_real), intent(inout), dimension(kts:kte), target :: bm - real(kind=c_real), intent(inout), dimension(kts:kte) :: bm_incld - - real(kind=c_real), intent(inout) :: precip_ice_surf - real(kind=c_real), intent(inout), dimension(kts:kte) :: qi_tend - real(kind=c_real), intent(inout), dimension(kts:kte) :: ni_tend - - call ice_sedimentation(kts,kte,ktop,kbot,kdir, & - rho,inv_rho,rhofaci,cld_frac_i,inv_dz,dt,inv_dt, & - qi,qi_incld,ni,qm,qm_incld,bm,bm_incld,ni_incld,precip_ice_surf,qi_tend,ni_tend) - - end subroutine ice_sedimentation_c - - subroutine rain_sedimentation_c(kts,kte,ktop,kbot,kdir, & - qr_incld,rho,inv_rho,rhofacr,cld_frac_r,inv_dz,dt,inv_dt, & - qr,nr,nr_incld,mu_r,lamr,precip_liq_surf,precip_liq_flux,qr_tend,nr_tend) bind(C) - use micro_p3, only: rain_sedimentation - - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - - real(kind=c_real), intent(in), dimension(kts:kte) :: rho - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_rho - real(kind=c_real), intent(in), dimension(kts:kte) :: rhofacr - real(kind=c_real), intent(in), dimension(kts:kte) :: cld_frac_r - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_dz - real(kind=c_real), value, intent(in) :: dt, inv_dt - - real(kind=c_real), intent(inout), target, dimension(kts:kte) :: qr - real(kind=c_real), intent(inout), target, dimension(kts:kte) :: nr - real(kind=c_real), intent(inout), dimension(kts:kte) :: qr_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: nr_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: mu_r - real(kind=c_real), intent(inout), dimension(kts:kte) :: lamr - real(kind=c_real), intent(inout) :: precip_liq_surf - real(kind=c_real), intent(inout), dimension(kts:kte+1) :: precip_liq_flux - real(kind=c_real), intent(inout), dimension(kts:kte) :: qr_tend - real(kind=c_real), intent(inout), dimension(kts:kte) :: nr_tend - - call rain_sedimentation(kts,kte,ktop,kbot,kdir, & - qr_incld,rho,inv_rho,rhofacr,cld_frac_r,inv_dz,dt,inv_dt, & - qr,nr,nr_incld,mu_r,lamr,precip_liq_surf,precip_liq_flux,qr_tend,nr_tend) - - end subroutine rain_sedimentation_c - - subroutine calc_bulk_rho_rime_c(qi_tot, qi_rim, bi_rim, rho_rime) bind(C) - use micro_p3, only: calc_bulkRhoRime - - ! arguments: - real(kind=c_real), value, intent(in) :: qi_tot - real(kind=c_real), intent(inout) :: qi_rim, bi_rim - real(kind=c_real), intent(out) :: rho_rime - - call calc_bulkRhoRime(qi_tot, qi_rim, bi_rim, rho_rime) - end subroutine calc_bulk_rho_rime_c - - subroutine homogeneous_freezing_c(kts,kte,ktop,kbot,kdir,T_atm,inv_exner,latent_heat_fusion, & - qc,nc,qr,nr,qi,ni,qm,bm,th_atm) bind(C) - use micro_p3, only: homogeneous_freezing - - ! arguments: - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - real(kind=c_real), intent(in), dimension(kts:kte) :: T_atm - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_exner - real(kind=c_real), intent(in), dimension(kts:kte) :: latent_heat_fusion - - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc - real(kind=c_real), intent(inout), dimension(kts:kte) :: qr - real(kind=c_real), intent(inout), dimension(kts:kte) :: nr - - real(kind=c_real), intent(inout), dimension(kts:kte) :: qi - real(kind=c_real), intent(inout), dimension(kts:kte) :: ni - real(kind=c_real), intent(inout), dimension(kts:kte) :: qm - real(kind=c_real), intent(inout), dimension(kts:kte) :: bm - real(kind=c_real), intent(inout), dimension(kts:kte) :: th_atm - - call homogeneous_freezing(kts,kte,ktop,kbot,kdir,T_atm,inv_exner,latent_heat_fusion, & - qc,nc,qr,nr,qi,ni,qm,bm,th_atm) - end subroutine homogeneous_freezing_c - - subroutine compute_rain_fall_velocity_c(qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr) bind(C) - use micro_p3, only: compute_rain_fall_velocity - - ! arguments: - real(kind=c_real), value, intent(in) :: qr_incld, rhofacr - real(kind=c_real), intent(inout) :: nr_incld - real(kind=c_real), intent(out) :: mu_r, lamr, V_qr, V_nr - - call compute_rain_fall_velocity(qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr) - end subroutine compute_rain_fall_velocity_c - -subroutine update_prognostic_ice_c(qc2qi_hetero_freeze_tend,qc2qi_collect_tend,qc2qr_ice_shed_tend,nc_collect_tend,nc2ni_immers_freeze_tend,ncshdc,qr2qi_collect_tend,nr_collect_tend,qr2qi_immers_freeze_tend,nr2ni_immers_freeze_tend,nr_ice_shed_tend, & - qi2qr_melt_tend,ni2nr_melt_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend,qv2qi_nucleat_tend,ni_nucleat_tend,ni_selfcollect_tend,ni_sublim_tend,qc2qi_berg_tend,inv_exner,latent_heat_sublim,latent_heat_fusion,do_predict_nc,log_wetgrowth, & - dt,nmltratio,rho_qm_cloud,th_atm,qv,qi,ni,qm,bm,qc,nc,qr,nr) bind(C) - use micro_p3, only: update_prognostic_ice - - ! arguments - real(kind=c_real), value, intent(in) :: qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend, ncshdc, qr2qi_collect_tend, nr_collect_tend, & - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, ni_nucleat_tend, ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, & - latent_heat_fusion, latent_heat_sublim, dt, nmltratio, rho_qm_cloud - - logical(kind=c_bool), value, intent(in) :: do_predict_nc, log_wetgrowth - - real(kind=c_real), intent(inout) :: th_atm, qv, qc, nc, qr, nr, qi, ni, qm, bm - - call update_prognostic_ice(qc2qi_hetero_freeze_tend,qc2qi_collect_tend,qc2qr_ice_shed_tend,nc_collect_tend,nc2ni_immers_freeze_tend,ncshdc,qr2qi_collect_tend,nr_collect_tend,qr2qi_immers_freeze_tend,nr2ni_immers_freeze_tend,nr_ice_shed_tend, & - qi2qr_melt_tend,ni2nr_melt_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend,qv2qi_nucleat_tend,ni_nucleat_tend,ni_selfcollect_tend,ni_sublim_tend,qc2qi_berg_tend,inv_exner,latent_heat_sublim,latent_heat_fusion,do_predict_nc,log_wetgrowth, & - dt,nmltratio,rho_qm_cloud,th_atm,qv,qi,ni,qm,bm,qc,nc,qr,nr) - - end subroutine update_prognostic_ice_c - - subroutine get_time_space_phys_variables_c(T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, & - ab, abi, kap, eii) bind(C) - use micro_p3, only: get_time_space_phys_variables - - !arguments - real(kind=c_real), value, intent(in) :: T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i - real(kind=c_real), intent(out) :: mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii - - call get_time_space_phys_variables(T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, & - ab, abi, kap, eii) - end subroutine get_time_space_phys_variables_c - - subroutine ice_cldliq_collection_c(rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, qc_incld, ni_incld, & - nc_incld, qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc) bind(C) - use micro_p3, only: ice_cldliq_collection - - ! arguments: - real(kind=c_real), value, intent(in) :: rho, temp, rhofaci, table_val_qc2qi_collect - real(kind=c_real), value, intent(in) :: qi_incld, qc_incld, ni_incld, nc_incld - real(kind=c_real), intent(out) :: qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc - - call ice_cldliq_collection(rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, qc_incld, ni_incld, & - nc_incld, qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc) - end subroutine ice_cldliq_collection_c - - subroutine ice_rain_collection_c(rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect, & - qi_incld, ni_incld, qr_incld, qr2qi_collect_tend, nr_collect_tend) bind(C) - use micro_p3, only: ice_rain_collection - - ! arguments: - real(kind=c_real), value, intent(in) :: rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect - real(kind=c_real), value, intent(in) :: qi_incld, ni_incld, qr_incld - real(kind=c_real), intent(out) :: qr2qi_collect_tend, nr_collect_tend - - call ice_rain_collection(rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect, & - qi_incld, ni_incld, qr_incld, qr2qi_collect_tend, nr_collect_tend) - end subroutine ice_rain_collection_c - - subroutine ice_self_collection_c(rho, rhofaci, table_val_ni_self_collect, eii, qm_incld, & - qi_incld, ni_incld, ni_selfcollect_tend) bind(C) - use micro_p3, only: ice_self_collection - - ! arguments: - real(kind=c_real), value, intent(in) :: rho, rhofaci, table_val_ni_self_collect, eii, qm_incld - real(kind=c_real), value, intent(in) :: qi_incld, ni_incld - real(kind=c_real), intent(out) :: ni_selfcollect_tend - - call ice_self_collection(rho, rhofaci, table_val_ni_self_collect, eii, qm_incld, & - qi_incld, ni_incld, ni_selfcollect_tend) - end subroutine ice_self_collection_c - - subroutine evaporate_rain_c(qr_incld,qc_incld,nr_incld,qi_incld, & - cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, & - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt,& - qr2qv_evap_tend,nr_evap_tend) bind(C) - use micro_p3, only: evaporate_rain - - ! arguments - real(kind=c_real), value, intent(in) :: qr_incld,qc_incld,nr_incld,qi_incld, & - cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, & - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt - - real(kind=c_real), intent(out) :: qr2qv_evap_tend, nr_evap_tend - - call evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, & - cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, & - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt,& - qr2qv_evap_tend,nr_evap_tend) - end subroutine evaporate_rain_c - - subroutine update_prognostic_liquid_c(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend,nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, & - qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, do_predict_nc, do_prescribed_CCN, inv_rho, inv_exner, latent_heat_vapor, dt, th_atm, qv, qc, nc, qr, nr) bind(C) - use micro_p3, only: update_prognostic_liquid - - ! arguments - real(kind=c_real), value, intent(in) :: qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, & - qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend - - logical(kind=c_bool), value, intent(in) :: do_predict_nc - logical(kind=c_bool), value, intent(in) :: do_prescribed_CCN - - real(kind=c_real), value, intent(in) :: inv_rho, inv_exner, latent_heat_vapor, dt - - real(kind=c_real), intent(inout) :: th_atm, qv, qc, nc, qr, nr - - call update_prognostic_liquid(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend,nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, & - qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, do_predict_nc, do_prescribed_CCN, inv_rho, inv_exner, latent_heat_vapor, dt, th_atm, qv, qc, nc, qr, nr) - - end subroutine update_prognostic_liquid_c - - subroutine ice_deposition_sublimation_c(qi_incld, ni_incld, t_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt, qidep, qi2qv_sublim_tend, ni_sublim_tend, qiberg) bind(C) - use micro_p3, only : ice_deposition_sublimation - - real(kind=c_real) , value, intent(in) :: qi_incld, ni_incld, t_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt - real(kind=c_real) , intent(out) :: qidep, qi2qv_sublim_tend, ni_sublim_tend, qiberg - - call ice_deposition_sublimation(qi_incld, ni_incld, t_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt, qidep, qi2qv_sublim_tend, ni_sublim_tend, qiberg) - end subroutine ice_deposition_sublimation_c - - subroutine ice_relaxation_timescale_c(rho, temp, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, & - dv, mu, sc, qi_incld, ni_incld, & - epsi, epsi_tot) bind(C) - use micro_p3, only: calc_ice_relaxation_timescale - - ! arguments - real(kind=c_real), value, intent(in) :: rho, temp, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, & - dv, mu, sc, qi_incld, ni_incld - real(kind=c_real), intent(out) :: epsi - real(kind=c_real), intent(inout) :: epsi_tot - - call calc_ice_relaxation_timescale(rho, temp, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, & - dv, mu, sc, qi_incld, ni_incld, & - epsi, epsi_tot) - end subroutine ice_relaxation_timescale_c - - subroutine calc_liq_relaxation_timescale_c(rho, f1r, f2r, dv, mu, sc, mu_r, & - lamr, cdistr, cdist, qr_incld, & - qc_incld, epsr, epsc) bind(C) - use micro_p3, only: calc_liq_relaxation_timescale - - ! arguments - real(kind=c_real), value, intent(in) :: rho,f1r,f2r,dv,mu,sc,mu_r,lamr, & - cdistr,cdist,qr_incld,qc_incld - real(kind=c_real), intent(out) :: epsr - real(kind=c_real), intent(out) :: epsc - - call calc_liq_relaxation_timescale(rho,f1r,f2r,dv,mu,sc,mu_r,lamr, & - cdistr,cdist,qr_incld,qc_incld,epsr, & - epsc) - end subroutine calc_liq_relaxation_timescale_c - - subroutine ice_nucleation_c(temp, inv_rho, ni, ni_activated, qv_supersat_i, inv_dt, & - do_predict_nc, do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend) bind(C) - use micro_p3, only: ice_nucleation - - ! arguments - real(kind=c_real), value, intent(in) :: temp, inv_rho, ni, ni_activated, qv_supersat_i, inv_dt - logical(c_bool), value, intent(in) :: do_predict_nc - logical(c_bool), value, intent(in) :: do_prescribed_CCN - - real(kind=c_real), intent(inout) :: qv2qi_nucleat_tend, ni_nucleat_tend - - call ice_nucleation(temp, inv_rho, ni, ni_activated, qv_supersat_i, inv_dt, & - do_predict_nc, do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend) - end subroutine ice_nucleation_c - - subroutine ice_melting_c(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion, & - dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend) bind(C) - use micro_p3, only: ice_melting - - ! arguments: - real(kind=c_real), value, intent(in) :: rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld - real(kind=c_real), intent(out) :: qi2qr_melt_tend,ni2nr_melt_tend - - call ice_melting(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend) - - end subroutine ice_melting_c - - subroutine ice_cldliq_wet_growth_c(rho, temp, pres, rhofaci, table_val_qi2qr_melting, & - table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, kap, mu, sc, qv, qc_incld, & - qi_incld, ni_incld, qr_incld, & - log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend) bind(C) - use micro_p3, only: ice_cldliq_wet_growth - - ! argmens - real(kind=c_real), value, intent(in) :: rho, temp ,pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, & - kap, mu, sc, qv, qc_incld, qi_incld, ni_incld,qr_incld - logical(kind=c_bool), intent(inout) :: log_wetgrowth - real(kind=c_real), intent(inout) :: qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend - - call ice_cldliq_wet_growth(rho, temp, pres, rhofaci, table_val_qi2qr_melting, & - table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, kap, mu, sc, qv, qc_incld, & - qi_incld, ni_incld, qr_incld, & - log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend) - end subroutine ice_cldliq_wet_growth_c - - subroutine get_latent_heat_c(its,ite,kts,kte,v,s,f) bind(C) - use micro_p3, only: get_latent_heat - - ! arguments - integer(kind=c_int), intent(in), value :: its, ite, kts, kte - real(kind=c_real), dimension(its:ite, kts:kte), intent(out) :: v, s, f - - call get_latent_heat(its,ite,kts,kte,v,s,f) - end subroutine get_latent_heat_c - - function subgrid_variance_scaling_c(relvar,expon) result(res) bind(C) - use micro_p3, only: subgrid_variance_scaling - - ! arguments - real(kind=c_real), value, intent(in) :: relvar,expon - real(kind=c_real) :: res - - res = subgrid_variance_scaling(relvar,expon) - return - end function subgrid_variance_scaling_c - - subroutine check_values_c(qv, temp, kts, kte, timestepcount, & - force_abort, source_ind, col_loc) bind(C) - use micro_p3, only: check_values - - ! argmens - real(kind=c_real), intent(in) :: qv(kts:kte), temp(kts:kte), col_loc(3) - integer(kind=c_int), value, intent(in) :: kts, kte, timestepcount, source_ind - logical(kind=c_bool), value, intent(in) :: force_abort - - call check_values(qv,Temp,kts,kte,timestepcount,force_abort,source_ind,col_loc) - end subroutine check_values_c - - subroutine calculate_incloud_mixingratios_c(qc, qr, qi, qm, nc, nr, ni, bm, & - inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, & - qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld) bind(C) - use micro_p3, only: calculate_incloud_mixingratios - - ! argumens - real(kind=c_real), value, intent(in) :: qc, qr, qi, qm, nc, nr, ni, bm, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r - real(kind=c_real), intent(inout) :: qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld - - call calculate_incloud_mixingratios(qc, qr, qi, qm, nc, nr, ni, bm, & - inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, & - qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld) - end subroutine calculate_incloud_mixingratios_c - - subroutine p3_main_part1_c(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, & - pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, & - qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld, is_nucleat_possible, is_hydromet_present) bind(C) - - use micro_p3, only: p3_main_part1 - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, kbot, ktop, kdir - logical(kind=c_bool), value, intent(in) :: do_predict_nc, do_prescribed_CCN - real(kind=c_real), value, intent(in) :: dt - - real(kind=c_real), intent(in), dimension(kts:kte) :: pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, & - inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed - - real(kind=c_real), intent(inout), dimension(kts:kte) :: T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, & - acn, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, & - qm_incld, nc_incld, nr_incld, ni_incld, bm_incld - - logical(kind=c_bool), intent(out) :: is_nucleat_possible, is_hydromet_present - - call p3_main_part1(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, & - pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, & - qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld, is_nucleat_possible, is_hydromet_present) - - end subroutine p3_main_part1_c - - subroutine p3_main_part2_c(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, & - pres, inv_exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, & - ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, qi, ni, & - qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, & - ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, & - nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, & - prctot, is_hydromet_present) bind(C) - - use micro_p3, only: p3_main_part2 - - !arguments - integer(kind=c_int), value, intent(in) :: kts, kte, kbot, ktop, kdir - logical(kind=c_bool), value, intent(in) :: do_predict_nc, do_prescribed_CCN - real(kind=c_real), value, intent(in) :: dt, inv_dt - - real(kind=c_real), intent(in), dimension(kts:kte) :: pres, inv_exner, inv_cld_frac_l, inv_cld_frac_i, & - inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev - - real(kind=c_real), intent(inout), dimension(kts:kte) :: T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofaci, acn, & - qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, & - qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, & - cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, & - vap_ice_exchange, liq_ice_exchange, pratot, prctot - - logical(kind=c_bool), intent(out) :: is_hydromet_present - - ! throwaway - real(kind=c_real), dimension(kts:kte,49) :: p3_tend_out - - call p3_main_part2(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, & - pres, inv_exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, qi, ni, & - qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, & - ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, & - nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, & - prctot, p3_tend_out, is_hydromet_present) - - end subroutine p3_main_part2_c - - subroutine p3_main_part3_c(kts, kte, kbot, ktop, kdir, & - inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, & - rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & - mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr) bind(C) - - use micro_p3, only: p3_main_part3 - - ! args - - integer(kind=c_int), value, intent(in) :: kts, kte, kbot, ktop, kdir - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_exner, cld_frac_l, cld_frac_r, cld_frac_i - real(kind=c_real), intent(inout), dimension(kts:kte) :: rho, inv_rho, rhofaci, & - qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & - mu_c, nu, lamc, mu_r, & - lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, & - diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr - - call p3_main_part3(kts, kte, kbot, ktop, kdir, & - inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, & - rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & - mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr) - - end subroutine p3_main_part3_c - - subroutine ice_supersat_conservation_c(qidep, qinuc, cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, dt, qi2qv_sublim_tend, qr2qv_evap_tend) bind(C) - use micro_p3, only : ice_supersat_conservation - - real(kind=c_real) , intent(inout) :: qidep, qinuc - real(kind=c_real) , value, intent(in) :: cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, dt, qi2qv_sublim_tend, qr2qv_evap_tend - - call ice_supersat_conservation(qidep, qinuc, cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, dt, qi2qv_sublim_tend, qr2qv_evap_tend) - end subroutine ice_supersat_conservation_c - subroutine nc_conservation_c(nc, nc_selfcollect_tend, dt, nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend) bind(C) - use micro_p3, only : nc_conservation - - real(kind=c_real) , value, intent(in) :: nc, nc_selfcollect_tend, dt - real(kind=c_real) , intent(inout) :: nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend - - call nc_conservation(nc, nc_selfcollect_tend, dt, nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend) - end subroutine nc_conservation_c - subroutine nr_conservation_c(nr, ni2nr_melt_tend, nr_ice_shed_tend, ncshdc, nc2nr_autoconv_tend, dt, nmltratio, nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend) bind(C) - use micro_p3, only : nr_conservation - - real(kind=c_real) , value, intent(in) :: nr, ni2nr_melt_tend, nr_ice_shed_tend, ncshdc, nc2nr_autoconv_tend, dt, nmltratio - real(kind=c_real) , intent(inout) :: nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend - - call nr_conservation(nr, ni2nr_melt_tend, nr_ice_shed_tend, ncshdc, nc2nr_autoconv_tend, dt, nmltratio, nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend) - end subroutine nr_conservation_c - subroutine ni_conservation_c(ni, ni_nucleat_tend, nr2ni_immers_freeze_tend, nc2ni_immers_freeze_tend, dt, ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend) bind(C) - use micro_p3, only : ni_conservation - - real(kind=c_real) , value, intent(in) :: ni, ni_nucleat_tend, nr2ni_immers_freeze_tend, nc2ni_immers_freeze_tend, dt - real(kind=c_real) , intent(inout) :: ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend - - call ni_conservation(ni, ni_nucleat_tend, nr2ni_immers_freeze_tend, nc2ni_immers_freeze_tend, dt, ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend) - end subroutine ni_conservation_c - subroutine prevent_liq_supersaturation_c(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend) bind(C) - use micro_p3, only : prevent_liq_supersaturation - - real(kind=c_real) , value, intent(in) :: pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc - real(kind=c_real) , intent(inout) :: qi2qv_sublim_tend, qr2qv_evap_tend - - call prevent_liq_supersaturation(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend) - end subroutine prevent_liq_supersaturation_c end module p3_iso_c diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/p3_main_wrap.cpp index ed46e281a67..eb10aca9e7f 100644 --- a/components/eamxx/src/physics/p3/p3_main_wrap.cpp +++ b/components/eamxx/src/physics/p3/p3_main_wrap.cpp @@ -8,53 +8,23 @@ using scream::Real; using scream::Int; -extern "C" { - void p3_main_c(Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, - Real* qv, Real dt, Real* qi, Real* qm, - Real* ni, Real* bm, Real* pres, - Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, - Int it, Real* precip_liq_surf, Real* precip_ice_surf, Int its, - Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, - Real* rho_qi, bool do_predict_nc, bool do_prescribed_CCN, Real* dpres, Real* inv_exner, - Real* qv2qi_depos_tend, - Real* precip_liq_flux, Real* precip_ice_flux, // 1 extra column size - Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, - Real* liq_ice_exchange, Real* vap_liq_exchange, - Real* vap_ice_exchange, Real* qv_prev, Real* t_prev, Real* elapsed_s); -} namespace scream { namespace p3 { -Int p3_main_wrap(const FortranData& d, bool use_fortran) { +Int p3_main_wrap(const FortranData& d) { EKAT_REQUIRE_MSG(d.dt > 0, "invalid dt"); - if (use_fortran) { - Real elapsed_s; - p3_main_c(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), - d.th_atm.data(), d.qv.data(), d.dt, d.qi.data(), - d.qm.data(), d.ni.data(), d.bm.data(), - d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), d.ni_activated.data(), d.inv_qc_relvar.data(), - d.it, d.precip_liq_surf.data(), d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, - d.diag_eff_radius_qc.data(), d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), - d.do_predict_nc, d.do_prescribed_CCN, d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), - d.precip_liq_flux.data(), d.precip_ice_flux.data(), d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), - d.liq_ice_exchange.data(), d.vap_liq_exchange.data(),d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data(), &elapsed_s); - return static_cast(elapsed_s * 1000000); - } - else { - return p3_main_f(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), d.th_atm.data(), - d.qv.data(), d.dt, d.qi.data(), d.qm.data(), d.ni.data(), - d.bm.data(), d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), - d.ni_activated.data(), d.inv_qc_relvar.data(), d.it, d.precip_liq_surf.data(), - d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, d.diag_eff_radius_qc.data(), - d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), d.do_predict_nc, d.do_prescribed_CCN, - d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), - d.precip_liq_flux.data(), d.precip_ice_flux.data(), - d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), - d.liq_ice_exchange.data(), d.vap_liq_exchange.data(), - d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data() ); - - } + return p3_main_host(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), d.th_atm.data(), + d.qv.data(), d.dt, d.qi.data(), d.qm.data(), d.ni.data(), + d.bm.data(), d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), + d.ni_activated.data(), d.inv_qc_relvar.data(), d.it, d.precip_liq_surf.data(), + d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, d.diag_eff_radius_qc.data(), + d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), d.do_predict_nc, d.do_prescribed_CCN, + d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), + d.precip_liq_flux.data(), d.precip_ice_flux.data(), + d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), + d.liq_ice_exchange.data(), d.vap_liq_exchange.data(), + d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data() ); } int test_p3_init () { @@ -63,11 +33,11 @@ int test_p3_init () { return 0; } -int test_p3_ic (bool use_fortran) { +int test_p3_ic () { const auto d = ic::Factory::create(ic::Factory::mixed); d->dt = 300.0; p3_init(); - p3_main_wrap(*d, use_fortran); + p3_main_wrap(*d); P3GlobalForFortran::deinit(); return 0; } diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.hpp b/components/eamxx/src/physics/p3/p3_main_wrap.hpp index 7c980fa8a5b..186b3247736 100644 --- a/components/eamxx/src/physics/p3/p3_main_wrap.hpp +++ b/components/eamxx/src/physics/p3/p3_main_wrap.hpp @@ -11,12 +11,11 @@ namespace p3 { struct FortranData; // Returns number of microseconds of p3_main execution -Int p3_main_wrap(const FortranData& d, bool use_fortran=false); +Int p3_main_wrap(const FortranData& d); int test_p3_init(); -int test_p3_ic(bool use_fortran); - +int test_p3_ic(); } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 66346279018..5a949afa249 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -57,17 +57,17 @@ endif() CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" LIBS p3 - EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" + EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3;physics") # Make sure that a diff in the two implementation triggers a failed test (in debug only) +# No need to run lots of different thread counts. if (SCREAM_ENABLE_BASELINE_TESTS) CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp LIBS p3 - EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" + EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3;physics;fail" ${FORCE_RUN_DIFF_FAILS}) endif() @@ -75,16 +75,16 @@ endif() if (NOT SCREAM_P3_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" LIBS p3_sk - EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" + EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3_sk;physics") # Make sure that a diff in the two implementation triggers a failed test (in debug only) + # No need to run lots of different thread counts. CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp LIBS p3_sk - EXE_ARGS "--flags='${BASELINE_FILE_ARG}'" + EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3_sk;physics;fail" ${FORCE_RUN_DIFF_FAILS}) endif() diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index 58066bb6eac..9df32bc32ff 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -44,10 +44,6 @@ void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - back_to_cell_average(back_to_cell_average_data[i]); - } // Read baseline data std::string baseline_name = this->m_baseline_path + "/back_to_cell_average.dat"; if (this->m_baseline_action == COMPARE) { diff --git a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp index 15d76d346f4..e7952c85032 100644 --- a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp @@ -41,7 +41,7 @@ void run_check_values_bfb() // Get data from cxx for (auto& d : cvd_cxx) { - check_values_f(d.qv, d.temp, d.kts, d.kte, d.timestepcount, d.force_abort, d.source_ind, d.col_loc); + check_values_host(d.qv, d.temp, d.kts, d.kte, d.timestepcount, d.force_abort, d.source_ind, d.col_loc); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index 29fbdad8aef..e3b325a6d1f 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -67,7 +67,7 @@ void run_bfb() // Get data from cxx for (auto& d : csds_cxx) { - cloud_sedimentation_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + cloud_sedimentation_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.qc_incld, d.rho, d.inv_rho, d.cld_frac_l, d.acn, d.inv_dz, d.dt, d.inv_dt, d.do_predict_nc, d.qc, d.nc, d.nc_incld, d.mu_c, d.lamc, &d.precip_liq_surf, d.qc_tend, d.nc_tend); diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index f63e0badfb0..9ddd97aa497 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -170,7 +170,7 @@ void run_bfb_ice_sed() // Get data from cxx for (auto& d : isds_cxx) { - ice_sedimentation_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + ice_sedimentation_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.rho, d.inv_rho, d.rhofaci, d.cld_frac_i, d.inv_dz, d.dt, d.inv_dt, d.qi, d.qi_incld, d.ni, d.qm, d.qm_incld, d.bm, d.bm_incld, @@ -253,7 +253,7 @@ void run_bfb_homogeneous_freezing() // Get data from cxx for (auto& d : hfds_cxx) { - homogeneous_freezing_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + homogeneous_freezing_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.T_atm, d.inv_exner, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.th_atm); } diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index 62a7e19dddb..b37978c6657 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -106,7 +106,7 @@ void run_bfb_p3_main_part1() // Get data from cxx for (auto& d : isds_cxx) { - p3_main_part1_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, + p3_main_part1_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, d.pres, d.dpres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.inv_exner, d.exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.T_atm, d.rho, d.inv_rho, d.qv_sat_l, d.qv_sat_i, d.qv_supersat_i, d.rhofacr, d.rhofaci, @@ -218,7 +218,7 @@ void run_bfb_p3_main_part2() // Get data from cxx for (auto& d : isds_cxx) { - p3_main_part2_f( + p3_main_part2_host( d.kts, d.kte, d.kbot, d.ktop, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, d.inv_dt, d.pres, d.dpres, d.dz, d.nc_nuceat_tend, d.inv_exner, d.exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.ni_activated, d.inv_qc_relvar, d.cld_frac_i, d.cld_frac_l, d.cld_frac_r, d.qv_prev, d.t_prev, @@ -345,7 +345,7 @@ void run_bfb_p3_main_part3() // Get data from cxx for (auto& d : isds_cxx) { - p3_main_part3_f( + p3_main_part3_host( d.kts, d.kte, d.kbot, d.ktop, d.kdir, d.inv_exner, d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, d.rho, d.inv_rho, d.rhofaci, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, @@ -458,7 +458,7 @@ void run_bfb_p3_main() // Get data from cxx for (auto& d : isds_cxx) { d.template transpose(); - p3_main_f( + p3_main_host( d.qc, d.nc, d.qr, d.nr, d.th_atm, d.qv, d.dt, d.qi, d.qm, d.ni, d.bm, d.pres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.ni_activated, d.inv_qc_relvar, d.it, d.precip_liq_surf, d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, d.diag_eff_radius_qr, diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index abad922cfb4..297d899a090 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -188,7 +188,7 @@ void run_bfb_rain_sed() #if defined(SCREAM_FORCE_RUN_DIFF) inv_dt *= 2; #endif - rain_sedimentation_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + rain_sedimentation_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.qr_incld, d.rho, d.inv_rho, d.rhofacr, d.cld_frac_r, d.inv_dz, d.dt, inv_dt, d.qr, d.nr, d.nr_incld, d.mu_r, d.lamr, &d.precip_liq_surf, d.precip_liq_flux, diff --git a/components/eamxx/src/physics/p3/tests/p3_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_tests.cpp index d1a64f48520..081252e9cff 100644 --- a/components/eamxx/src/physics/p3/tests/p3_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_tests.cpp @@ -29,13 +29,8 @@ TEST_CASE("p3_init", "p3") { REQUIRE(nerr == 0); } -TEST_CASE("p3_ic_f", "p3") { - int nerr = scream::p3::test_p3_ic(true); - REQUIRE(nerr == 0); -} - TEST_CASE("p3_ic_c", "p3") { - int nerr = scream::p3::test_p3_ic(false); + int nerr = scream::p3::test_p3_ic(); REQUIRE(nerr == 0); } diff --git a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp index b6b5307966a..4c91cfe8fa9 100644 --- a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp @@ -250,7 +250,7 @@ void run_bfb() for (auto& d : cuds_cxx) { Real** fluxes, **vs, **qnx; d.convert_to_ptr_arr(tmp1, fluxes, vs, qnx); - calc_first_order_upwind_step_f( + calc_first_order_upwind_step_host( d.kts, d.kte, d.kdir, d.kbot, d.k_qxtop, d.dt_sub, d.rho, d.inv_rho, d.inv_dz, d.num_arrays, fluxes, vs, qnx); @@ -335,7 +335,7 @@ void run_bfb() for (auto& d : gsds_cxx) { Real** fluxes, **vs, **qnx; d.convert_to_ptr_arr(tmp1, fluxes, vs, qnx); - generalized_sedimentation_f(d.kts, d.kte, d.kdir, d.k_qxtop, + generalized_sedimentation_host(d.kts, d.kte, d.kdir, d.k_qxtop, &d.k_qxbot, d.kbot, d.Co_max, &d.dt_left, &d.prt_accum, d.inv_dz, d.inv_rho, d.rho, d.num_arrays, fluxes, vs, qnx); From 576d7b4f045bf0758db8480c3affda99cd468c2a Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 4 Nov 2024 15:37:16 -0700 Subject: [PATCH 226/366] Rename FortranData --- components/eamxx/src/physics/p3/p3_f90.cpp | 14 +++++------ components/eamxx/src/physics/p3/p3_f90.hpp | 24 +++++++++---------- .../eamxx/src/physics/p3/p3_functions_f90.cpp | 2 +- .../eamxx/src/physics/p3/p3_functions_f90.hpp | 10 ++++---- .../eamxx/src/physics/p3/p3_ic_cases.cpp | 8 +++---- .../eamxx/src/physics/p3/p3_ic_cases.hpp | 4 ++-- .../eamxx/src/physics/p3/p3_main_wrap.cpp | 2 +- .../eamxx/src/physics/p3/p3_main_wrap.hpp | 4 ++-- .../p3/tests/p3_ice_tables_unit_tests.cpp | 2 +- .../src/physics/p3/tests/p3_run_and_cmp.cpp | 14 +++++------ .../eamxx/src/physics/p3/tests/p3_tests.cpp | 8 +++---- 11 files changed, 46 insertions(+), 46 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_f90.cpp b/components/eamxx/src/physics/p3/p3_f90.cpp index 650ad3c645d..bf19af693e5 100644 --- a/components/eamxx/src/physics/p3/p3_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_f90.cpp @@ -16,7 +16,7 @@ extern "C" { namespace scream { namespace p3 { -FortranData::FortranData (Int ncol_, Int nlev_) +P3Data::P3Data (Int ncol_, Int nlev_) : ncol(ncol_), nlev(nlev_) { do_predict_nc = true; @@ -62,11 +62,11 @@ FortranData::FortranData (Int ncol_, Int nlev_) vap_ice_exchange = Array2("sum of vap-ice phase change tendenices", ncol, nlev); } -FortranDataIterator::FortranDataIterator (const FortranData::Ptr& d) { +P3DataIterator::P3DataIterator (const P3Data::Ptr& d) { init(d); } -void FortranDataIterator::init (const FortranData::Ptr& dp) { +void P3DataIterator::init (const P3Data::Ptr& dp) { d_ = dp; #define fdipb(name) \ fields_.push_back({#name, \ @@ -87,8 +87,8 @@ void FortranDataIterator::init (const FortranData::Ptr& dp) { #undef fdipb } -const FortranDataIterator::RawArray& -FortranDataIterator::getfield (Int i) const { +const P3DataIterator::RawArray& +P3DataIterator::getfield (Int i) const { EKAT_ASSERT(i >= 0 || i < nfield()); return fields_[i]; } @@ -112,8 +112,8 @@ void p3_init (const bool write_tables, const bool masterproc) { } } -int test_FortranData () { - FortranData d(11, 72); +int test_P3Data () { + P3Data d(11, 72); return 0; } diff --git a/components/eamxx/src/physics/p3/p3_f90.hpp b/components/eamxx/src/physics/p3/p3_f90.hpp index d07524d9c7a..fceca0b5601 100644 --- a/components/eamxx/src/physics/p3/p3_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_f90.hpp @@ -10,8 +10,8 @@ namespace scream { namespace p3 { // Data format we can use to communicate with Fortran version. -struct FortranData { - typedef std::shared_ptr Ptr; +struct P3Data { + typedef std::shared_ptr Ptr; using KT = KokkosTypes; using Scalar = Real; @@ -36,29 +36,29 @@ struct FortranData { Array3 p3_tend_out; Array2 liq_ice_exchange,vap_liq_exchange,vap_ice_exchange; - FortranData(Int ncol, Int nlev); + P3Data(Int ncol, Int nlev); }; -// Iterate over a FortranData's arrays. For examples, see Baseline::write, read. -struct FortranDataIterator { +// Iterate over a P3Data's arrays. For examples, see Baseline::write, read. +struct P3DataIterator { struct RawArray { std::string name; Int dim; Int extent[3]; - FortranData::Scalar* data; - FortranData::Array1::size_type size; + P3Data::Scalar* data; + P3Data::Array1::size_type size; }; - explicit FortranDataIterator(const FortranData::Ptr& d); + explicit P3DataIterator(const P3Data::Ptr& d); Int nfield () const { return fields_.size(); } const RawArray& getfield(Int i) const; private: - FortranData::Ptr d_; + P3Data::Ptr d_; std::vector fields_; - void init(const FortranData::Ptr& d); + void init(const P3Data::Ptr& d); }; void p3_init(const bool write_tables = false, @@ -68,9 +68,9 @@ void p3_init(const bool write_tables = false, // to the exact implementation or arithmetic in P3. For now, these checks are // here to establish that the initial regression-testing code gives results that // match the python f2py tester, without needing a data file. -Int check_against_python(const FortranData& d); +Int check_against_python(const P3Data& d); -int test_FortranData(); +int test_P3Data(); } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index 40d5342a31f..19d9b59e6b8 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -21,7 +21,7 @@ void p3_init_a_c(Real* ice_table_vals, Real* collect_table_vals); namespace scream { namespace p3 { -void p3_init_a(P3InitAFortranData& d) +void p3_init_a(P3InitAP3Data& d) { p3_init(); // need to initialize p3 first so that tables are loaded p3_init_a_c(d.ice_table_vals.data(), d.collect_table_vals.data()); diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 7042f0f9fb8..a1bbe864f4c 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -15,7 +15,7 @@ namespace p3 { /////////////////////////////////////////////////////////////////////////////// -struct P3InitAFortranData +struct P3InitAP3Data { // Must use Host as device, f90 code might not be able to use Device memory using P3F = Functions; @@ -28,9 +28,9 @@ struct P3InitAFortranData view_ice_table ice_table_vals; view_collect_table collect_table_vals; - P3InitAFortranData() : - ice_table_vals("P3InitAFortranData::ice_table_vals"), - collect_table_vals("P3InitAFortranData::collect_table_vals") + P3InitAP3Data() : + ice_table_vals("P3InitAP3Data::ice_table_vals"), + collect_table_vals("P3InitAP3Data::collect_table_vals") {} }; @@ -882,7 +882,7 @@ struct PreventLiqSupersaturationData { PTD_RW_SCALARS_ONLY(2, qi2qv_sublim_tend, qr2qv_evap_tend); }; -void p3_init_a(P3InitAFortranData& d); +void p3_init_a(P3InitAP3Data& d); /** * Convenience functions for calling p3 routines from the host with scalar data. diff --git a/components/eamxx/src/physics/p3/p3_ic_cases.cpp b/components/eamxx/src/physics/p3/p3_ic_cases.cpp index 1560dc0afb2..52b70de6025 100644 --- a/components/eamxx/src/physics/p3/p3_ic_cases.cpp +++ b/components/eamxx/src/physics/p3/p3_ic_cases.cpp @@ -8,12 +8,12 @@ namespace p3 { namespace ic { // From mixed_case_data.py in scream-docs at commit 4bbea4. -FortranData::Ptr make_mixed (const Int ncol, const Int nlev) { +P3Data::Ptr make_mixed (const Int ncol, const Int nlev) { using consts = scream::physics::Constants; const Int nk = nlev; Int k; - const auto dp = std::make_shared(ncol, nk); + const auto dp = std::make_shared(ncol, nk); auto& d = *dp; for (Int i = 0; i < ncol; ++i) { @@ -66,7 +66,7 @@ FortranData::Ptr make_mixed (const Int ncol, const Int nlev) { // To get potential temperature, start by making absolute temperature vary // between 150K at top of atmos and 300k at surface, then convert to potential // temp. - FortranData::Array1 T_atm("T", nk); + P3Data::Array1 T_atm("T", nk); for (k = 0; k < nk; ++k) { T_atm(k) = 150 + 150/double(nk)*k; if (i > 0) T_atm(k) += ((i % 3) - 0.5)/double(nk)*k; @@ -119,7 +119,7 @@ FortranData::Ptr make_mixed (const Int ncol, const Int nlev) { return dp; } -FortranData::Ptr Factory::create (IC ic, Int ncol, Int nlev) { +P3Data::Ptr Factory::create (IC ic, Int ncol, Int nlev) { switch (ic) { case mixed: return make_mixed(ncol, nlev); default: diff --git a/components/eamxx/src/physics/p3/p3_ic_cases.hpp b/components/eamxx/src/physics/p3/p3_ic_cases.hpp index a8c461b06da..0ab502e8c00 100644 --- a/components/eamxx/src/physics/p3/p3_ic_cases.hpp +++ b/components/eamxx/src/physics/p3/p3_ic_cases.hpp @@ -7,12 +7,12 @@ namespace scream { namespace p3 { namespace ic { -FortranData::Ptr make_mixed(Int ncol); +P3Data::Ptr make_mixed(Int ncol); struct Factory { enum IC { mixed }; - static FortranData::Ptr create(IC ic, Int ncol = 1, Int nlev = 72); + static P3Data::Ptr create(IC ic, Int ncol = 1, Int nlev = 72); }; } // namespace ic diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/p3_main_wrap.cpp index eb10aca9e7f..27b510bcbc6 100644 --- a/components/eamxx/src/physics/p3/p3_main_wrap.cpp +++ b/components/eamxx/src/physics/p3/p3_main_wrap.cpp @@ -12,7 +12,7 @@ using scream::Int; namespace scream { namespace p3 { -Int p3_main_wrap(const FortranData& d) { +Int p3_main_wrap(const P3Data& d) { EKAT_REQUIRE_MSG(d.dt > 0, "invalid dt"); return p3_main_host(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), d.th_atm.data(), d.qv.data(), d.dt, d.qi.data(), d.qm.data(), d.ni.data(), diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.hpp b/components/eamxx/src/physics/p3/p3_main_wrap.hpp index 186b3247736..c55007427cd 100644 --- a/components/eamxx/src/physics/p3/p3_main_wrap.hpp +++ b/components/eamxx/src/physics/p3/p3_main_wrap.hpp @@ -8,10 +8,10 @@ namespace scream { namespace p3 { -struct FortranData; +struct P3Data; // Returns number of microseconds of p3_main execution -Int p3_main_wrap(const FortranData& d); +Int p3_main_wrap(const P3Data& d); int test_p3_init(); diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp index 98f61766860..4acd18f2927 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp @@ -32,7 +32,7 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base Functions::init_kokkos_ice_lookup_tables(ice_table_vals, collect_table_vals); // Get data from fortran - P3InitAFortranData d; + P3InitAP3Data d; p3_init_a(d); // Copy device data to host diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index 9e491cea6a4..91b7f255048 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -35,10 +35,10 @@ using namespace scream::p3; * large discrepancies. */ Int compare (const double& tol, - const FortranData::Ptr& ref, const FortranData::Ptr& d) { + const P3Data::Ptr& ref, const P3Data::Ptr& d) { Int nerr = 0; - FortranDataIterator refi(ref), di(d); + P3DataIterator refi(ref), di(d); EKAT_ASSERT(refi.nfield() == di.nfield()); for (Int i = 0, n = refi.nfield(); i < n; ++i) { const auto& fr = refi.getfield(i); @@ -170,7 +170,7 @@ struct Baseline { bool do_predict_nc, do_prescribed_CCN; }; - static void set_params (const ParamSet& ps, FortranData& d) { + static void set_params (const ParamSet& ps, P3Data& d) { // Items not set by factory d.dt = ps.dt; d.it = ps.nsteps; @@ -180,8 +180,8 @@ struct Baseline { std::vector params_; - static void write (const ekat::FILEPtr& fid, const FortranData::Ptr& d) { - FortranDataIterator fdi(d); + static void write (const ekat::FILEPtr& fid, const P3Data::Ptr& d) { + P3DataIterator fdi(d); for (Int i = 0, n = fdi.nfield(); i < n; ++i) { const auto& f = fdi.getfield(i); ekat::write(&f.dim, 1, fid); @@ -190,8 +190,8 @@ struct Baseline { } } - static void read (const ekat::FILEPtr& fid, const FortranData::Ptr& d) { - FortranDataIterator fdi(d); + static void read (const ekat::FILEPtr& fid, const P3Data::Ptr& d) { + P3DataIterator fdi(d); for (Int i = 0, n = fdi.nfield(); i < n; ++i) { const auto& f = fdi.getfield(i); int dim, ds[3]; diff --git a/components/eamxx/src/physics/p3/tests/p3_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_tests.cpp index 081252e9cff..cbb00623cd0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_tests.cpp @@ -5,15 +5,15 @@ namespace { -TEST_CASE("FortranData", "p3") { - int val = scream::p3::test_FortranData(); +TEST_CASE("P3Data", "p3") { + int val = scream::p3::test_P3Data(); REQUIRE(val == 0); } -TEST_CASE("FortranDataIterator", "p3") { +TEST_CASE("P3DataIterator", "p3") { using scream::p3::ic::Factory; const auto d = Factory::create(Factory::mixed); - scream::p3::FortranDataIterator fdi(d); + scream::p3::P3DataIterator fdi(d); REQUIRE(fdi.nfield() == 35); const auto& f = fdi.getfield(0); REQUIRE(f.dim == 2); From 96f5a284f09cc0f5faf1ebf2c2bd59a66ec69dad Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Mon, 4 Nov 2024 15:54:24 -0800 Subject: [PATCH 227/366] fix typo in new changes --- components/eamxx/src/share/grid/mesh_free_grids_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index b2878881fc4..446fba5229f 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -290,7 +290,7 @@ load_vertical_coordinates (const nonconstgrid_ptr_type& grid, const std::string& auto hybi_v = hybi.get_view(); auto ilev_v = ilev.get_view(); auto num_lev = grid->get_num_vertical_levels(); - for (int ii=0;iinum_lev;ii++) { + for (int ii=0;ii Date: Mon, 4 Nov 2024 23:26:43 -0600 Subject: [PATCH 228/366] Adding support for PIO_IOTYPE_ADIOSC Adding support for a new SCORPIO I/O type, adiosc, that supports compressing data (lossless compression) written out using ADIOS (BP file format) To enable data compression using ADIOS, users need to change (xmlchange) PIO_TYPENAME to "adiosc" --- cime_config/machines/config_machines.xml | 3 ++- driver-mct/cime_config/config_component.xml | 2 +- driver-moab/cime_config/config_component.xml | 2 +- share/build/buildlib.spio | 8 ++++++++ share/util/shr_pio_mod.F90 | 2 ++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 36381d78153..bd39d417fc1 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1987,7 +1987,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /nfs/gce/projects/climate/software/perl5/lib/perl5 - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.9.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.10.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$BLOSC2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/c-blosc2/2.15.1/gcc-12.1.0; else echo "$BLOSC2_ROOT"; fi} diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index 8cddad0abf8..cdfca40befa 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -2628,7 +2628,7 @@ char - netcdf,pnetcdf,netcdf4p,netcdf4c,adios,hdf5,default + netcdf,pnetcdf,netcdf4p,netcdf4c,adios,adiosc,hdf5,default run_pio env_run.xml pio io type diff --git a/driver-moab/cime_config/config_component.xml b/driver-moab/cime_config/config_component.xml index d3685126c61..ac9e46a476e 100644 --- a/driver-moab/cime_config/config_component.xml +++ b/driver-moab/cime_config/config_component.xml @@ -2620,7 +2620,7 @@ char - netcdf,pnetcdf,netcdf4p,netcdf4c,adios,hdf5,default + netcdf,pnetcdf,netcdf4p,netcdf4c,adios,adiosc,hdf5,default run_pio env_run.xml pio io type diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index d17627c4323..064fa751e4a 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -78,6 +78,8 @@ def buildlib(bldroot, installpath, case): # ADIOS2_ROOT. This is a workaround if "ADIOS2_ROOT" in os.environ: os.environ["ADIOS2_DIR"] = os.environ["ADIOS2_ROOT"] + if "BLOSC2_ROOT" in os.environ: + os.environ["Blosc2_DIR"] = os.environ["BLOSC2_ROOT"] # If variable PIO_VERSION_MAJOR is defined in the environment then # we assume that PIO is installed on the system @@ -105,10 +107,14 @@ def buildlib(bldroot, installpath, case): else: cmake_opts += f"-DGENF90_PATH={scorpio_src_dir}/src/genf90 " + adiosc_found = False if "ADIOS2_ROOT" in os.environ: cmake_opts += "-DWITH_ADIOS2:BOOL=ON " if "FROM_CREATE_TEST" in os.environ and os.environ["FROM_CREATE_TEST"] == "True": cmake_opts += "-DADIOS_BP2NC_TEST:BOOL=ON " + if "BLOSC2_ROOT" in os.environ: + cmake_opts += "-DADIOS_USE_COMPRESSION:BOOL=ON " + adiosc_found = True if debug: cmake_opts += "-DPIO_ENABLE_LOGGING=ON " @@ -281,6 +287,8 @@ def buildlib(bldroot, installpath, case): valid_values += ",netcdf4p,netcdf4c" if adios_found: valid_values += ",adios" + if adiosc_found: + valid_values += ",adiosc" if hdf5_found: valid_values += ",hdf5" diff --git a/share/util/shr_pio_mod.F90 b/share/util/shr_pio_mod.F90 index d500cbc680a..11cefd725b5 100644 --- a/share/util/shr_pio_mod.F90 +++ b/share/util/shr_pio_mod.F90 @@ -679,6 +679,8 @@ subroutine shr_pio_getiotypefromname(typename, iotype, defaulttype) #ifndef PIO1 else if ( typename .eq. 'ADIOS') then iotype = pio_iotype_adios + else if ( typename .eq. 'ADIOSC') then + iotype = pio_iotype_adiosc else if ( typename .eq. 'HDF5') then iotype = pio_iotype_hdf5 #endif From a1c1f07cdf407e63adc8247fb4be171124587eea Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Tue, 5 Nov 2024 09:23:07 -0600 Subject: [PATCH 229/366] Adding a testmod to force ADIOS BP compression Adding a testmod to use/force data compression for ADIOS output --- cime_config/testmods_dirs/io/force_adiosc/shell_commands | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 cime_config/testmods_dirs/io/force_adiosc/shell_commands diff --git a/cime_config/testmods_dirs/io/force_adiosc/shell_commands b/cime_config/testmods_dirs/io/force_adiosc/shell_commands new file mode 100644 index 00000000000..543d5dbfef0 --- /dev/null +++ b/cime_config/testmods_dirs/io/force_adiosc/shell_commands @@ -0,0 +1,2 @@ +#!/bin/bash +./xmlchange PIO_TYPENAME="adiosc" From 312ee63150dfe0e975b087f3c02eda4d7d834ed7 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 5 Nov 2024 09:39:16 -0700 Subject: [PATCH 230/366] reorg --- components/eamxx/src/physics/p3/CMakeLists.txt | 14 -------------- .../src/physics/p3/eamxx_p3_process_interface.cpp | 2 +- .../eamxx/src/physics/p3/p3_tables_setup.cpp | 2 +- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 14 ++++++++------ .../p3/{p3_f90.cpp => tests/infra/p3_data.cpp} | 2 +- .../p3/{p3_f90.hpp => tests/infra/p3_data.hpp} | 6 +++--- .../physics/p3/{ => tests/infra}/p3_ic_cases.cpp | 0 .../physics/p3/{ => tests/infra}/p3_ic_cases.hpp | 2 +- .../physics/p3/{ => tests/infra}/p3_main_wrap.cpp | 4 ++-- .../physics/p3/{ => tests/infra}/p3_main_wrap.hpp | 0 .../infra/p3_test_data.cpp} | 4 ++-- .../infra/p3_test_data.hpp} | 0 .../p3/tests/p3_autoconversion_unit_tests.cpp | 2 +- .../tests/p3_back_to_cell_average_unit_tests.cpp | 2 +- ...p3_calc_liq_relaxation_timescale_unit_tests.cpp | 2 +- .../p3/tests/p3_calc_rime_density_unit_tests.cpp | 2 +- .../p3/tests/p3_check_values_unit_tests.cpp | 4 ++-- .../p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp | 2 +- .../p3/tests/p3_cloud_rain_acc_unit_tests.cpp | 2 +- .../physics/p3/tests/p3_cloud_sed_unit_tests.cpp | 2 +- .../p3/tests/p3_droplet_self_coll_unit_tests.cpp | 2 +- .../src/physics/p3/tests/p3_dsd2_unit_tests.cpp | 2 +- .../p3/tests/p3_evaporate_rain_unit_tests.cpp | 2 +- .../src/physics/p3/tests/p3_find_unit_tests.cpp | 2 +- .../tests/p3_ice_cldliq_wet_growth_unit_tests.cpp | 2 +- .../p3/tests/p3_ice_collection_unit_tests.cpp | 2 +- .../tests/p3_ice_deposition_sublimation_tests.cpp | 2 +- .../physics/p3/tests/p3_ice_melting_unit_tests.cpp | 2 +- .../p3/tests/p3_ice_nucleation_unit_tests.cpp | 2 +- .../p3_ice_relaxation_timescale_unit_tests.cpp | 2 +- .../src/physics/p3/tests/p3_ice_sed_unit_tests.cpp | 2 +- .../tests/p3_ice_supersat_conservation_tests.cpp | 2 +- .../physics/p3/tests/p3_ice_tables_unit_tests.cpp | 2 +- .../tests/p3_incloud_mixingratios_unit_tests.cpp | 2 +- .../src/physics/p3/tests/p3_main_unit_tests.cpp | 2 +- .../physics/p3/tests/p3_nc_conservation_tests.cpp | 2 +- .../physics/p3/tests/p3_ni_conservation_tests.cpp | 2 +- .../physics/p3/tests/p3_nr_conservation_tests.cpp | 2 +- .../tests/p3_prevent_liq_supersaturation_tests.cpp | 2 +- .../p3/tests/p3_rain_imm_freezing_unit_tests.cpp | 2 +- .../physics/p3/tests/p3_rain_sed_unit_tests.cpp | 4 ++-- .../p3/tests/p3_rain_self_collection_tests.cpp | 2 +- .../eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp | 2 +- .../p3_subgrid_variance_scaling_unit_tests.cpp | 2 +- .../src/physics/p3/tests/p3_table3_unit_tests.cpp | 4 ++-- components/eamxx/src/physics/p3/tests/p3_tests.cpp | 2 +- .../eamxx/src/physics/p3/tests/p3_unit_tests.cpp | 2 +- .../src/physics/p3/tests/p3_unit_tests_common.hpp | 4 ++-- .../src/physics/p3/tests/p3_upwind_unit_tests.cpp | 2 +- 49 files changed, 60 insertions(+), 72 deletions(-) rename components/eamxx/src/physics/p3/{p3_f90.cpp => tests/infra/p3_data.cpp} (99%) rename components/eamxx/src/physics/p3/{p3_f90.hpp => tests/infra/p3_data.hpp} (93%) rename components/eamxx/src/physics/p3/{ => tests/infra}/p3_ic_cases.cpp (100%) rename components/eamxx/src/physics/p3/{ => tests/infra}/p3_ic_cases.hpp (93%) rename components/eamxx/src/physics/p3/{ => tests/infra}/p3_main_wrap.cpp (96%) rename components/eamxx/src/physics/p3/{ => tests/infra}/p3_main_wrap.hpp (100%) rename components/eamxx/src/physics/p3/{p3_functions_f90.cpp => tests/infra/p3_test_data.cpp} (99%) rename components/eamxx/src/physics/p3/{p3_functions_f90.hpp => tests/infra/p3_test_data.hpp} (100%) diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index a1dfc946267..df1285412a5 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -1,19 +1,10 @@ set(P3_SRCS - p3_f90.cpp - p3_ic_cases.cpp p3_iso_c.f90 ${SCREAM_BASE_DIR}/../eam/src/physics/p3/scream/micro_p3.F90 eamxx_p3_process_interface.cpp eamxx_p3_run.cpp ) -if (NOT SCREAM_LIB_ONLY) - list(APPEND P3_SRCS - p3_functions_f90.cpp - p3_main_wrap.cpp - ) # Add f90 bridges needed for testing -endif() - # Add ETI source files if not on CUDA/HIP if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) list(APPEND P3_SRCS @@ -123,11 +114,6 @@ endforeach() add_executable(p3_tables_setup EXCLUDE_FROM_ALL p3_tables_setup.cpp) target_link_libraries(p3_tables_setup p3) -#crusher change -if (Kokkos_ENABLE_HIP) -set_source_files_properties(p3_functions_f90.cpp PROPERTIES COMPILE_FLAGS -O0) -endif() - if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index a2b8ccb2e29..9dfdb385a4d 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -3,7 +3,7 @@ #include "share/property_checks/field_lower_bound_check.hpp" // Needed for p3_init, the only F90 code still used. #include "physics/p3/p3_functions.hpp" -#include "physics/p3/p3_f90.hpp" +#include "physics/p3/p3_data.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_units.hpp" diff --git a/components/eamxx/src/physics/p3/p3_tables_setup.cpp b/components/eamxx/src/physics/p3/p3_tables_setup.cpp index 8582674d1fe..ec3f5ccbb43 100644 --- a/components/eamxx/src/physics/p3/p3_tables_setup.cpp +++ b/components/eamxx/src/physics/p3/p3_tables_setup.cpp @@ -1,6 +1,6 @@ // This is a tiny program that calls p3_init() to generate tables used by p3 -#include "physics/p3/p3_f90.hpp" +#include "physics/p3/p3_data.hpp" int main(int /* argc */, char** /* argv */) { scream::p3::p3_init(/* write_tables = */ true); diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 5a949afa249..70ac3947fa8 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -1,5 +1,7 @@ include(ScreamUtils) +add_subdirectory(infra) + set(P3_TESTS_SRCS p3_tests.cpp p3_unit_tests.cpp @@ -56,7 +58,7 @@ else() endif() CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" - LIBS p3 + LIBS p3 p3_test_infra EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3;physics") @@ -65,7 +67,7 @@ CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" # No need to run lots of different thread counts. if (SCREAM_ENABLE_BASELINE_TESTS) CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3 + LIBS p3 p3_test_infra EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF LABELS "p3;physics;fail" @@ -74,7 +76,7 @@ endif() if (NOT SCREAM_P3_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" - LIBS p3_sk + LIBS p3_sk p3_test_infra EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3_sk;physics") @@ -82,7 +84,7 @@ if (NOT SCREAM_P3_SMALL_KERNELS) # Make sure that a diff in the two implementation triggers a failed test (in debug only) # No need to run lots of different thread counts. CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3_sk + LIBS p3_sk p3_test_infra EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF LABELS "p3_sk;physics;fail" @@ -90,7 +92,7 @@ if (NOT SCREAM_P3_SMALL_KERNELS) endif() CreateUnitTest(p3_run_and_cmp "p3_run_and_cmp.cpp" - LIBS p3 + LIBS p3 p3_test_infra EXCLUDE_MAIN_CPP THREADS ${SCREAM_TEST_MAX_THREADS} EXE_ARGS "${BASELINE_FILE_ARG}" @@ -99,7 +101,7 @@ CreateUnitTest(p3_run_and_cmp "p3_run_and_cmp.cpp" # Make sure that a diff from baselines triggers a failed test (in debug only) if (SCREAM_ENABLE_BASELINE_TESTS) CreateUnitTest(p3_run_and_cmp_fail "p3_run_and_cmp.cpp" - LIBS p3 + LIBS p3 p3_test_infra COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF THREADS ${SCREAM_TEST_MAX_THREADS} EXE_ARGS "${BASELINE_FILE_ARG}" diff --git a/components/eamxx/src/physics/p3/p3_f90.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp similarity index 99% rename from components/eamxx/src/physics/p3/p3_f90.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_data.cpp index bf19af693e5..793c062f63f 100644 --- a/components/eamxx/src/physics/p3/p3_f90.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp @@ -1,4 +1,4 @@ -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "physics_constants.hpp" #include "p3_ic_cases.hpp" diff --git a/components/eamxx/src/physics/p3/p3_f90.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp similarity index 93% rename from components/eamxx/src/physics/p3/p3_f90.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_data.hpp index fceca0b5601..d3319445e7c 100644 --- a/components/eamxx/src/physics/p3/p3_f90.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp @@ -1,5 +1,5 @@ -#ifndef SCREAM_P3_F90_HPP -#define SCREAM_P3_F90_HPP +#ifndef SCREAM_P3_DATA_HPP +#define SCREAM_P3_DATA_HPP #include "share/scream_types.hpp" @@ -9,7 +9,7 @@ namespace scream { namespace p3 { -// Data format we can use to communicate with Fortran version. +// Data format we can use to store (and read/write) data for a full P3 run. struct P3Data { typedef std::shared_ptr Ptr; diff --git a/components/eamxx/src/physics/p3/p3_ic_cases.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.cpp similarity index 100% rename from components/eamxx/src/physics/p3/p3_ic_cases.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.cpp diff --git a/components/eamxx/src/physics/p3/p3_ic_cases.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.hpp similarity index 93% rename from components/eamxx/src/physics/p3/p3_ic_cases.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.hpp index 0ab502e8c00..bbf6f133e51 100644 --- a/components/eamxx/src/physics/p3/p3_ic_cases.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.hpp @@ -1,7 +1,7 @@ #ifndef INCLUDE_SCREAM_P3_IC_CASES_HPP #define INCLUDE_SCREAM_P3_IC_CASES_HPP -#include "p3_f90.hpp" +#include "p3_data.hpp" namespace scream { namespace p3 { diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp similarity index 96% rename from components/eamxx/src/physics/p3/p3_main_wrap.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp index 27b510bcbc6..4126d80c034 100644 --- a/components/eamxx/src/physics/p3/p3_main_wrap.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp @@ -1,6 +1,6 @@ #include "p3_main_wrap.hpp" -#include "p3_f90.hpp" -#include "p3_functions_f90.hpp" +#include "p3_data.hpp" +#include "p3_test_data.hpp" #include "physics_constants.hpp" #include "p3_ic_cases.hpp" diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.hpp similarity index 100% rename from components/eamxx/src/physics/p3/p3_main_wrap.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.hpp diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp similarity index 99% rename from components/eamxx/src/physics/p3/p3_functions_f90.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp index 19d9b59e6b8..d0b25286987 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp @@ -1,6 +1,6 @@ -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "ekat/ekat_pack_kokkos.hpp" diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp similarity index 100% rename from components/eamxx/src/physics/p3/p3_functions_f90.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp diff --git a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp index 39a8d143544..cb799a67287 100644 --- a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index 9df32bc32ff..aa70e7fe42c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index c0da609087c..7f3d7ee126f 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp index 1fee033460b..ef4cd621843 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp index e7952c85032..2a858f86dfc 100644 --- a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp @@ -4,8 +4,8 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" +#include "p3_test_data.hpp" +#include "p3_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp index 9c152443405..5dacc132a87 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp index 276187f8ec0..fff6784ab61 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index e3b325a6d1f..4f729d8fde6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp index 2a96b1b3bff..b17d8309c3c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 6563335acb0..5bf199fa85a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index a17165bafdd..15efcd85fca 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp index 9c812f8f842..6e8e09226b7 100644 --- a/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index 5060d9d2fdc..3389b52efc6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index 6f34526e9b0..a2f3e7a8b72 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp index 7a52c1b42f0..e5983fb8c74 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index 86c42707c1e..f3e57a415bd 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp index 154adbabb15..47115afd430 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp index 7015b7a94f8..08bded63ff4 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index 9ddd97aa497..c250a15a6ff 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp index c15c1383ab7..0e56fb75bec 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp index 4acd18f2927..92097e57191 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp index fd768e70839..2741992b90e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index b37978c6657..136fdb2fbe2 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp index e13d49affc0..26b9beecc7e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp index 2f2db25d1d7..6c50154c046 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp index 5b8b0d83951..b33dd02b0f0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index c4f95b28c83..31d18572f3b 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -3,7 +3,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "share/scream_types.hpp" #include "physics/share/physics_functions.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp index 92123d63e82..875b47e4722 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index 297d899a090..eac5c4827b6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -4,8 +4,8 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" +#include "p3_test_data.hpp" +#include "p3_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp index d9d8ef00412..c175f691547 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index 91b7f255048..30195147cac 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -3,7 +3,7 @@ #include "share/util/scream_utils.hpp" #include "p3_main_wrap.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_ic_cases.hpp" #include "ekat/util/ekat_file_utils.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp index a71b3abd422..bf1c3543dd2 100644 --- a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp @@ -3,7 +3,7 @@ #include "share/scream_types.hpp" #include "ekat/ekat_pack.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp index cc14bfc46d2..57197fea6c8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp @@ -3,8 +3,8 @@ #include "p3_unit_tests_common.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" +#include "p3_test_data.hpp" +#include "p3_data.hpp" #include "share/scream_types.hpp" #include "ekat/ekat_pack.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_tests.cpp index cbb00623cd0..858a231094d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_tests.cpp @@ -1,5 +1,5 @@ #include "catch2/catch.hpp" -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "p3_main_wrap.hpp" #include "p3_ic_cases.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index 2c1971f0ba9..c3f8571d117 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -5,7 +5,7 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "ekat/util/ekat_arch.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp index 685284a0463..68ff4569eb1 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp @@ -3,9 +3,9 @@ #include "share/scream_types.hpp" #include "p3_functions.hpp" -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "ekat/util/ekat_test_utils.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include #include diff --git a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp index 4c91cfe8fa9..1f65984d727 100644 --- a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp @@ -3,7 +3,7 @@ #include "p3_unit_tests_common.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" From 3c2c875e80bd6c211d2a4f97afd3cc0a4dfe0158 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 5 Nov 2024 10:49:27 -0700 Subject: [PATCH 231/366] More reorg --- .../eamxx/src/physics/p3/CMakeLists.txt | 1 + .../physics/p3/eamxx_p3_process_interface.cpp | 10 ++--- .../eamxx/src/physics/p3/eti/p3_init.cpp | 14 +++++++ .../src/physics/p3/impl/p3_init_impl.hpp | 40 +++++++++++++++++++ .../eamxx/src/physics/p3/p3_functions.hpp | 4 ++ components/eamxx/src/physics/p3/p3_iso_c.f90 | 9 ----- .../src/physics/p3/tests/infra/CMakeLists.txt | 15 +++++++ .../src/physics/p3/tests/infra/p3_data.cpp | 25 ------------ .../src/physics/p3/tests/infra/p3_data.hpp | 3 -- .../physics/p3/tests/infra/p3_main_wrap.cpp | 9 ++++- .../physics/p3/tests/infra/p3_test_data.cpp | 4 +- .../src/physics/p3/tests/p3_run_and_cmp.cpp | 7 ++-- .../physics/p3/tests/p3_unit_tests_common.hpp | 2 +- 13 files changed, 93 insertions(+), 50 deletions(-) create mode 100644 components/eamxx/src/physics/p3/eti/p3_init.cpp create mode 100644 components/eamxx/src/physics/p3/impl/p3_init_impl.hpp create mode 100644 components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index df1285412a5..2a630438049 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -18,6 +18,7 @@ if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos eti/p3_table_ice.cpp eti/p3_dsd2.cpp eti/p3_find.cpp + eti/p3_init.cpp eti/p3_update_prognostics.cpp eti/p3_get_time_space_phys_variables.cpp eti/p3_autoconversion.cpp diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index 9dfdb385a4d..f6771d6bf17 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -1,9 +1,7 @@ -#include "physics/p3/eamxx_p3_process_interface.hpp" #include "share/property_checks/field_within_interval_check.hpp" #include "share/property_checks/field_lower_bound_check.hpp" -// Needed for p3_init, the only F90 code still used. -#include "physics/p3/p3_functions.hpp" -#include "physics/p3/p3_data.hpp" +#include "p3_functions.hpp" +#include "eamxx_p3_process_interface.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_units.hpp" @@ -243,8 +241,8 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) add_postcondition_check(get_field_out("eff_radius_qr"),m_grid,0.0,5.0e3,false); // Initialize p3 - p3::p3_init(/* write_tables = */ false, - this->get_comm().am_i_root()); + P3F::p3_init(/* write_tables = */ false, + this->get_comm().am_i_root()); // Initialize all of the structures that are passed to p3_main in run_impl. // Note: Some variables in the structures are not stored in the field manager. For these diff --git a/components/eamxx/src/physics/p3/eti/p3_init.cpp b/components/eamxx/src/physics/p3/eti/p3_init.cpp new file mode 100644 index 00000000000..b1878af8984 --- /dev/null +++ b/components/eamxx/src/physics/p3/eti/p3_init.cpp @@ -0,0 +1,14 @@ +#include "p3_init_impl.hpp" + +namespace scream { +namespace p3 { + +/* + * Explicit instantiation for doing find functions on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace p3 +} // namespace scream diff --git a/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp new file mode 100644 index 00000000000..9b4b999bce0 --- /dev/null +++ b/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp @@ -0,0 +1,40 @@ +#ifndef P3_INIT_IMPL_HPP +#define P3_INIT_IMPL_HPP + +#include "p3_functions.hpp" // for ETI only but harmless for GPU + +extern "C" { + void micro_p3_utils_init_c(scream::Real Cpair, scream::Real Rair, scream::Real RH2O, scream::Real RHO_H2O, + scream::Real MWH2O, scream::Real MWdry, scream::Real gravit, scream::Real LatVap, scream::Real LatIce, + scream::Real CpLiq, scream::Real Tmelt, scream::Real Pi, bool masterproc); + void p3_init_c(const char** lookup_file_dir, int* info, const bool& write_tables); +} + +namespace scream { +namespace p3 { + +/* + * Implementation of p3 init. Clients should NOT #include + * this file, #include p3_functions.hpp instead. + */ +template +void Functions +::p3_init (const bool write_tables, const bool masterproc) { + static bool is_init = false; + if (!is_init) { + using c = scream::physics::Constants; + micro_p3_utils_init_c(c::Cpair, c::Rair, c::RH2O, c::RHO_H2O, + c::MWH2O, c::MWdry, c::gravit, c::LatVap, c::LatIce, + c::CpLiq, c::Tmelt, c::Pi, masterproc); + static const char* dir = SCREAM_DATA_DIR "/tables"; + Int info; + p3_init_c(&dir, &info, write_tables); + EKAT_REQUIRE_MSG(info == 0, "p3_init_c returned info " << info); + is_init = true; + } +} + +} // namespace p3 +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 4ac88b4afc4..2aaaf5afb3e 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -357,6 +357,9 @@ struct Functions static void init_kokkos_ice_lookup_tables( view_ice_table& ice_table_vals, view_collect_table& collect_table_vals); + static void p3_init(const bool write_tables = false, + const bool masterproc = false); + // Map (mu_r, lamr) to Table3 data. KOKKOS_FUNCTION static void lookup(const Spack& mu_r, const Spack& lamr, @@ -1468,5 +1471,6 @@ void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, # include "p3_nr_conservation_impl.hpp" # include "p3_ni_conservation_impl.hpp" # include "p3_prevent_liq_supersaturation_impl.hpp" +# inlcude "p3_init_impl.hpp" #endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE #endif // P3_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index 34ee99cde33..71c846b7167 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -14,15 +14,6 @@ module p3_iso_c ! contains - subroutine append_precision(string, prefix) - - character(kind=c_char, len=512), intent(out) :: string - character(*), intent(in) :: prefix - real(kind=c_real) :: s - - write (string, '(a,i1,a1)') prefix, sizeof(s), C_NULL_CHAR - end subroutine append_precision - subroutine init_tables_from_f90_c(vn_table_vals_c, vm_table_vals_c, revap_table_vals_c, mu_table_c) bind(C) use micro_p3, only: p3_get_tables diff --git a/components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt new file mode 100644 index 00000000000..6f7093a33ec --- /dev/null +++ b/components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt @@ -0,0 +1,15 @@ +set(INFRA_SRCS + p3_data.cpp + p3_ic_cases.cpp + p3_main_wrap.cpp + p3_test_data.cpp +) + +#crusher change +if (Kokkos_ENABLE_HIP) +set_source_files_properties(p3_test_data.cpp PROPERTIES COMPILE_FLAGS -O0) +endif() + +add_library(p3_test_infra ${INFRA_SRCS}) +target_link_libraries(p3_test_infra p3) +target_include_directories(p3_test_infra PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp index 793c062f63f..24fd6529f22 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp @@ -6,12 +6,6 @@ using scream::Real; using scream::Int; -extern "C" { - void micro_p3_utils_init_c(Real Cpair, Real Rair, Real RH2O, Real RHO_H2O, - Real MWH2O, Real MWdry, Real gravit, Real LatVap, Real LatIce, - Real CpLiq, Real Tmelt, Real Pi, bool masterproc); - void p3_init_c(const char** lookup_file_dir, int* info, const bool& write_tables); -} namespace scream { namespace p3 { @@ -93,25 +87,6 @@ P3DataIterator::getfield (Int i) const { return fields_[i]; } -void micro_p3_utils_init (const bool masterproc) { - using c = scream::physics::Constants; - micro_p3_utils_init_c(c::Cpair, c::Rair, c::RH2O, c::RHO_H2O, - c::MWH2O, c::MWdry, c::gravit, c::LatVap, c::LatIce, - c::CpLiq, c::Tmelt, c::Pi, masterproc); -} - -void p3_init (const bool write_tables, const bool masterproc) { - static bool is_init = false; - if (!is_init) { - micro_p3_utils_init(masterproc); - static const char* dir = SCREAM_DATA_DIR "/tables"; - Int info; - p3_init_c(&dir, &info, write_tables); - EKAT_REQUIRE_MSG(info == 0, "p3_init_c returned info " << info); - is_init = true; - } -} - int test_P3Data () { P3Data d(11, 72); return 0; diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp index d3319445e7c..df5b25e311a 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp @@ -61,9 +61,6 @@ struct P3DataIterator { void init(const P3Data::Ptr& d); }; -void p3_init(const bool write_tables = false, - const bool masterproc = false); - // We will likely want to remove these checks in the future, as we're not tied // to the exact implementation or arithmetic in P3. For now, these checks are // here to establish that the initial regression-testing code gives results that diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp index 4126d80c034..2b758a513ec 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp @@ -28,15 +28,20 @@ Int p3_main_wrap(const P3Data& d) { } int test_p3_init () { - p3_init(); + using P3F = Functions; + + P3F::p3_init(); P3GlobalForFortran::deinit(); return 0; } int test_p3_ic () { + using P3F = Functions; + const auto d = ic::Factory::create(ic::Factory::mixed); + d->dt = 300.0; - p3_init(); + P3F::p3_init(); p3_main_wrap(*d); P3GlobalForFortran::deinit(); return 0; diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp index d0b25286987..23a9998f43c 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp @@ -23,7 +23,9 @@ namespace p3 { void p3_init_a(P3InitAP3Data& d) { - p3_init(); // need to initialize p3 first so that tables are loaded + using P3F = Functions; + + P3F::p3_init(); // need to initialize p3 first so that tables are loaded p3_init_a_c(d.ice_table_vals.data(), d.collect_table_vals.data()); } diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index 30195147cac..4c0d5a214e0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -16,6 +16,7 @@ namespace { using namespace scream; using namespace scream::p3; +using P3F = Functions; /* * p3_run_and_cmp can be run in 2 modes. First, generate_baseline @@ -87,7 +88,7 @@ struct Baseline { for (Int r = -1; r < ps.repeat; ++r) { const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); set_params(ps, *d); - p3_init(); + P3F::p3_init(); if (ps.repeat > 0 && r == -1) { std::cout << "Running P3 with ni=" << d->ncol << ", nk=" << d->nlev @@ -130,7 +131,7 @@ struct Baseline { if (no_baseline) { const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); set_params(ps, *d); - p3_init(); + P3F::p3_init(); for (int it=0; itfirst; std::stringstream ss(raw_flags); From 6caafa33b10f4009426239581650ddcfe65b905c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 5 Nov 2024 11:05:39 -0700 Subject: [PATCH 232/366] Fix typo --- components/eamxx/src/physics/p3/p3_functions.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 2aaaf5afb3e..eef5a8ec73e 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -1471,6 +1471,6 @@ void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, # include "p3_nr_conservation_impl.hpp" # include "p3_ni_conservation_impl.hpp" # include "p3_prevent_liq_supersaturation_impl.hpp" -# inlcude "p3_init_impl.hpp" +# include "p3_init_impl.hpp" #endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE #endif // P3_FUNCTIONS_HPP From 00f42a77df75eb2c7f5e6b46934f12c8f9e7cdae Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Wed, 11 Sep 2024 13:29:47 -0500 Subject: [PATCH 233/366] Add ICOS30 Wave Mesh. --- cime_config/config_grids.xml | 27 +++++++++++++++++++ .../namelist_defaults_ww3_grid.xml | 6 ++--- .../namelist_defaults_ww3_grid_nml.xml | 4 +-- components/ww3/cime_config/buildnml | 3 ++- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index ddb1df949ac..715db0fb704 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -2405,6 +2405,16 @@ EC30to60E2r2 + + ne30np4.pg2 + r05 + IcoswISC30E3r5 + r05 + null + wQU225Icos30E3r5 + IcoswISC30E3r5 + + ne30np4.pg2 ne30np4.pg2 @@ -3476,6 +3486,13 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.wQU225EC30to60E2r2.220224.nc WW3 unstructured QU 225km global grid with EC30to60E2r2 coastlines + + + 97988 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.wQU225Icos30E3r5.240910.nc + WW3 unstructured QU 225km global grid with ICOS30 coastlines + @@ -5042,6 +5059,10 @@ cpl/gridmaps/wQU225EC30to60E2r2/map_ne30pg2_TO_wQU225EC30to60E2r2_blin.20220222.nc + + cpl/gridmaps/wQU225Icos30E3r5/map_ne30pg2_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + + cpl/gridmaps/wQU225EC30to60E2r2/map_TL319_TO_wQU225EC30to60E2r2_blin.20220602.nc @@ -5058,6 +5079,12 @@ cpl/gridmaps/wQU225EC30to60E2r2/map_EC30to60E2r2_TO_wQU225EC30to60E2r2_blin.20220222.nc + + cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfbilin.20240910.nc + cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + + cpl/gridmaps/wQU225EC60to30/map_CFSv2_TO_wQU225EC60to30_blin.20210412.nc diff --git a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml index e8bb1d42b24..0e4d4674087 100644 --- a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml +++ b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml @@ -11,9 +11,9 @@ variables. Values that depend on the model configuration use attributes to express the dependency. --> -1.07 +1.1 0.035 -50 +36 36 0.5 @@ -30,7 +30,7 @@ attributes to express the dependency. 450.0 30.0 -wQU225EC30to60E2r2 +wQU225Icos30E3r5 ww3_grid_namelists.nml UNST SPHE diff --git a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml index 9573b482a45..9ebe2b4e505 100644 --- a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml +++ b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml @@ -158,8 +158,8 @@ attributes to express the dependency. -obstructions_local.glo_unst.in -obstructions_shadow.glo_unst.in +obstructions_local.rtd.in +obstructions_shadow.rtd.in 1.00 1.00 diff --git a/components/ww3/cime_config/buildnml b/components/ww3/cime_config/buildnml index 3fe5d79d3f8..c43a521c29c 100755 --- a/components/ww3/cime_config/buildnml +++ b/components/ww3/cime_config/buildnml @@ -55,7 +55,8 @@ def buildnml(case, caseroot, compname): "wQU225EC60to30sp50x36", "wQU225EC30to60E2r2sp50x36", "wQU225EC30to60E2r2sp36x36", - "wQU225EC30to60E2r2sp25x36") + "wQU225EC30to60E2r2sp25x36", + "wQU225Icos30E3r5sp36x36") expect((wav_grid+wav_spec) in wav_grid_supported, "Combination of WAV_GRID {} and WAV_SPEC {} is not supported in ww3. Choose from: '{}'".format(wav_grid,wav_spec,wav_grid_supported) ) #-------------------------------------------------------------------- From 158d9cec3008839b2733a77b04aebebbacc82586 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Wed, 25 Sep 2024 12:44:26 -0500 Subject: [PATCH 234/366] mask out sotkes drift under ice shelves for icos mesh. --- components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F index 87fb36f2145..d1fdc41167c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F @@ -141,6 +141,8 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim integer, dimension(:), pointer :: & maxLevelCell, minLevelCell, nEdgesOnCell, maxLevelEdgeTop, minLevelEdgeBot + integer, dimension(:), pointer :: landIceMask + real (kind=RKIND), dimension(:), pointer :: & latCell, lonCell, bottomDepth, fCell, & ssh, dcEdge, dvEdge, areaCell, iceFraction, & @@ -241,6 +243,7 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim ! set pointers for fields related to ocean forcing state ! call mpas_pool_get_array(forcingPool, 'iceFraction', iceFraction) + call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) call mpas_pool_get_array(forcingPool, 'windSpeed10m', windSpeed10m) call mpas_pool_get_array(forcingPool, 'windStressZonal', windStressZonal) call mpas_pool_get_array(forcingPool, 'windStressMeridional', windStressMeridional) @@ -489,7 +492,8 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim surfaceFrictionVelocity(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params) - else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then + else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND & + .and. landIceMask(iCell) .eq. 0) then call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & windStressMeridional(iCell), & surfaceFrictionVelocity(iCell), & @@ -684,7 +688,8 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim surfaceFrictionVelocity(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params) - else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then + else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND & + .and. landIceMask(iCell) .eq. 0) then call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & windStressMeridional(iCell), & surfaceFrictionVelocity(iCell), & From 8cbd06e8e16a4970e8159771f0329ddb1b2ac38e Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Thu, 12 Sep 2024 11:34:04 -0500 Subject: [PATCH 235/366] add wave-ice remapping files for wQU225Icos30 --- cime_config/config_grids.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 715db0fb704..2f2c7ff2647 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -3653,8 +3653,12 @@ ATM2ROF_FMAPNAME ATM2ROF_SMAPNAME ATM2WAV_SMAPNAME + WAV2ATM_SMAPNAME OCN2WAV_SMAPNAME + WAV2OCN_SMAPNAME + WAV2OCN_FMAPNAME ICE2WAV_SMAPNAME + WAV2ICE_SMAPNAME ROF2OCN_LIQ_RMAPNAME ROF2OCN_ICE_RMAPNAME @@ -5061,6 +5065,7 @@ cpl/gridmaps/wQU225Icos30E3r5/map_ne30pg2_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_ne30pg2_esmfbilin.20240910.nc @@ -5074,13 +5079,17 @@ + cpl/gridmaps/wQU225EC30to60E2r2/map_wQU225EC30to60E2r2_TO_EC30to60E2r2_aave.20220222.nc cpl/gridmaps/wQU225EC30to60E2r2/map_wQU225EC30to60E2r2_TO_EC30to60E2r2_blin.20220222.nc + cpl/gridmaps/wQU225EC30to60E2r2/map_wQU225EC30to60E2r2_TO_EC30to60E2r2_blin.20220222.nc cpl/gridmaps/wQU225EC30to60E2r2/map_EC30to60E2r2_TO_wQU225EC30to60E2r2_blin.20220222.nc cpl/gridmaps/wQU225EC30to60E2r2/map_EC30to60E2r2_TO_wQU225EC30to60E2r2_blin.20220222.nc + cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfaave.20240910.nc cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfbilin.20240910.nc + cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfbilin.20240910.nc cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc From b9a8485674b3639b11568b72afa57de1058b6a26 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Thu, 12 Sep 2024 11:45:31 -0500 Subject: [PATCH 236/366] add wav-ocean flux,wav-ice remapping file definitions for ICOS wave mesh --- driver-mct/cime_config/config_component.xml | 34 +++++++ .../cime_config/namelist_definition_drv.xml | 92 ++++++++++++++++++- 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index 8cddad0abf8..22383ce6003 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -1518,6 +1518,23 @@ atm2wav state mapping file decomp type + + char + idmap + run_domain + env_run.xml + wav2atm state mapping file + + + + char + X,Y + Y + run_domain + env_run.xml + wav2atm state mapping file decomp type + + char idmap @@ -2002,6 +2019,23 @@ wav2ocn state mapping file decomp type + + char + idmap + run_domain + env_run.xml + wav2ocn flux mapping file + + + + char + X,Y + X + run_domain + env_run.xml + wav2ocn flux mapping file decomp type + + char idmap diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index 982acabe64d..933a75bceac 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -4891,7 +4891,7 @@ X - + char mapping @@ -4952,6 +4952,36 @@ + + char + mapping + abs + seq_maps + + atm to wav state mapping file for states + + + $WAV2ATM_SMAPNAME + + + + + char + mapping + seq_maps + + The type of mapping desired, either "source" or "destination" mapping. + X is associated with rearrangement of the source grid to the + destination grid and then local mapping. Y is associated with mapping + on the source grid and then rearrangement and sum to the destination + grid. + + + $WAV2ATM_SMAPTYPE + X + + + char mapping @@ -4982,6 +5012,66 @@ + + char + mapping + abs + seq_maps + + wav to ocn flux mapping file for fluxes + + + $WAV2OCN_FMAPNAME + + + + + char + mapping + seq_maps + + The type of mapping desired, either "source" or "destination" mapping. + X is associated with rearrangement of the source grid to the + destination grid and then local mapping. Y is associated with mapping + on the source grid and then rearrangement and sum to the destination + grid. + + + $WAV2OCN_FMAPTYPE + X + + + + + char + mapping + abs + seq_maps + + wav to ice state mapping file for states + + + $WAV2ICE_SMAPNAME + + + + + char + mapping + seq_maps + + The type of mapping desired, either "source" or "destination" mapping. + X is associated with rearrangement of the source grid to the + destination grid and then local mapping. Y is associated with mapping + on the source grid and then rearrangement and sum to the destination + grid. + + + $WAV2ICE_SMAPTYPE + X + + + char(10) drv_physics From c8b9aa516b68aed97b45173f85050b42939a0f92 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Wed, 25 Sep 2024 12:52:09 -0500 Subject: [PATCH 237/366] add wave to ocean FLUX remapper --- driver-mct/main/prep_ocn_mod.F90 | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/driver-mct/main/prep_ocn_mod.F90 b/driver-mct/main/prep_ocn_mod.F90 index b726b1516b2..f87b566c6f8 100644 --- a/driver-mct/main/prep_ocn_mod.F90 +++ b/driver-mct/main/prep_ocn_mod.F90 @@ -66,6 +66,7 @@ module prep_ocn_mod public :: prep_ocn_get_mapper_Sg2o public :: prep_ocn_get_mapper_Fg2o public :: prep_ocn_get_mapper_Sw2o + public :: prep_ocn_get_mapper_Fw2o !-------------------------------------------------------------------------- ! Private interfaces @@ -90,6 +91,7 @@ module prep_ocn_mod type(seq_map), pointer :: mapper_Fg2o type(seq_map), pointer :: mapper_Sg2o type(seq_map), pointer :: mapper_Sw2o + type(seq_map), pointer :: mapper_Fw2o ! attribute vectors type(mct_aVect), pointer :: a2x_ox(:) ! Atm export, ocn grid, cpl pes @@ -190,6 +192,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc allocate(mapper_Sg2o) allocate(mapper_Fg2o) allocate(mapper_Sw2o) + allocate(mapper_Fw2o) if (ocn_present) then @@ -382,6 +385,13 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc call seq_map_init_rcfile(mapper_Sw2o, wav(1), ocn(1), & 'seq_maps.rc', 'wav2ocn_smapname:', 'wav2ocn_smaptype:',samegrid_ow, & 'mapper_Sw2o initialization') + if (iamroot_CPLID) then + write(logunit,*) ' ' + write(logunit,F00) 'Initializing mapper_Fw2o' + end if + call seq_map_init_rcfile(mapper_Fw2o, wav(1), ocn(1), & + 'seq_maps.rc', 'wav2ocn_fmapname:', 'wav2ocn_fmaptype:',samegrid_ow, & + 'mapper_Fw2o initialization') endif call shr_sys_flush(logunit) @@ -1446,7 +1456,8 @@ subroutine prep_ocn_calc_w2x_ox(timer) call t_drvstartf (trim(timer),barrier=mpicom_CPLID) do ewi = 1,num_inst_wav w2x_wx => component_get_c2x_cx(wav(ewi)) - call seq_map_map(mapper_Sw2o, w2x_wx, w2x_ox(ewi), norm=.true.) + call seq_map_map(mapper_Sw2o, w2x_wx, w2x_ox(ewi),fldlist=seq_flds_w2x_states, norm=.true.) + call seq_map_map(mapper_Fw2o, w2x_wx, w2x_ox(ewi),fldlist=seq_flds_w2x_fluxes, norm=.true.) enddo call t_drvstopf (trim(timer)) end subroutine prep_ocn_calc_w2x_ox @@ -1547,5 +1558,10 @@ function prep_ocn_get_mapper_Sw2o() type(seq_map), pointer :: prep_ocn_get_mapper_Sw2o prep_ocn_get_mapper_Sw2o => mapper_Sw2o end function prep_ocn_get_mapper_Sw2o + + function prep_ocn_get_mapper_Fw2o() + type(seq_map), pointer :: prep_ocn_get_mapper_Fw2o + prep_ocn_get_mapper_Fw2o => mapper_Fw2o + end function prep_ocn_get_mapper_Fw2o end module prep_ocn_mod From a8e5d2d81bc9ac2db3446f57e7c8ce0c834c147e Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Mon, 30 Sep 2024 16:19:38 -0500 Subject: [PATCH 238/366] remove wav-atm, wav-ice coupling (for future PR) --- cime_config/config_grids.xml | 5 -- driver-mct/cime_config/config_component.xml | 17 ------ .../cime_config/namelist_definition_drv.xml | 60 ------------------- 3 files changed, 82 deletions(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 2f2c7ff2647..13fff49e823 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -3653,12 +3653,10 @@ ATM2ROF_FMAPNAME ATM2ROF_SMAPNAME ATM2WAV_SMAPNAME - WAV2ATM_SMAPNAME OCN2WAV_SMAPNAME WAV2OCN_SMAPNAME WAV2OCN_FMAPNAME ICE2WAV_SMAPNAME - WAV2ICE_SMAPNAME ROF2OCN_LIQ_RMAPNAME ROF2OCN_ICE_RMAPNAME @@ -5065,7 +5063,6 @@ cpl/gridmaps/wQU225Icos30E3r5/map_ne30pg2_to_wQU225Icos30E3r5_esmfbilin.20240910.nc - cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_ne30pg2_esmfbilin.20240910.nc @@ -5081,7 +5078,6 @@ cpl/gridmaps/wQU225EC30to60E2r2/map_wQU225EC30to60E2r2_TO_EC30to60E2r2_aave.20220222.nc cpl/gridmaps/wQU225EC30to60E2r2/map_wQU225EC30to60E2r2_TO_EC30to60E2r2_blin.20220222.nc - cpl/gridmaps/wQU225EC30to60E2r2/map_wQU225EC30to60E2r2_TO_EC30to60E2r2_blin.20220222.nc cpl/gridmaps/wQU225EC30to60E2r2/map_EC30to60E2r2_TO_wQU225EC30to60E2r2_blin.20220222.nc cpl/gridmaps/wQU225EC30to60E2r2/map_EC30to60E2r2_TO_wQU225EC30to60E2r2_blin.20220222.nc @@ -5089,7 +5085,6 @@ cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfaave.20240910.nc cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfbilin.20240910.nc - cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfbilin.20240910.nc cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index 22383ce6003..b917809f6fa 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -1518,23 +1518,6 @@ atm2wav state mapping file decomp type - - char - idmap - run_domain - env_run.xml - wav2atm state mapping file - - - - char - X,Y - Y - run_domain - env_run.xml - wav2atm state mapping file decomp type - - char idmap diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index 933a75bceac..efb24402e78 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -4952,36 +4952,6 @@ - - char - mapping - abs - seq_maps - - atm to wav state mapping file for states - - - $WAV2ATM_SMAPNAME - - - - - char - mapping - seq_maps - - The type of mapping desired, either "source" or "destination" mapping. - X is associated with rearrangement of the source grid to the - destination grid and then local mapping. Y is associated with mapping - on the source grid and then rearrangement and sum to the destination - grid. - - - $WAV2ATM_SMAPTYPE - X - - - char mapping @@ -5042,36 +5012,6 @@ - - char - mapping - abs - seq_maps - - wav to ice state mapping file for states - - - $WAV2ICE_SMAPNAME - - - - - char - mapping - seq_maps - - The type of mapping desired, either "source" or "destination" mapping. - X is associated with rearrangement of the source grid to the - destination grid and then local mapping. Y is associated with mapping - on the source grid and then rearrangement and sum to the destination - grid. - - - $WAV2ICE_SMAPTYPE - X - - - char(10) drv_physics From 2b820160aaeb0cc927600b2b0df5e0f7b514ceb9 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 1 Oct 2024 10:51:36 -0500 Subject: [PATCH 239/366] remove seq_map_map call for wave to ocn fluxes (no fluxes coupled yet) --- driver-mct/main/prep_ocn_mod.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/driver-mct/main/prep_ocn_mod.F90 b/driver-mct/main/prep_ocn_mod.F90 index f87b566c6f8..6577f1c6034 100644 --- a/driver-mct/main/prep_ocn_mod.F90 +++ b/driver-mct/main/prep_ocn_mod.F90 @@ -1457,7 +1457,6 @@ subroutine prep_ocn_calc_w2x_ox(timer) do ewi = 1,num_inst_wav w2x_wx => component_get_c2x_cx(wav(ewi)) call seq_map_map(mapper_Sw2o, w2x_wx, w2x_ox(ewi),fldlist=seq_flds_w2x_states, norm=.true.) - call seq_map_map(mapper_Fw2o, w2x_wx, w2x_ox(ewi),fldlist=seq_flds_w2x_fluxes, norm=.true.) enddo call t_drvstopf (trim(timer)) end subroutine prep_ocn_calc_w2x_ox From f346119098588ad6e895aa4fe01aa5f83ce27a13 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 1 Oct 2024 11:16:43 -0500 Subject: [PATCH 240/366] add B case ICOS test to wav_developer test suite --- cime_config/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cime_config/tests.py b/cime_config/tests.py index 37812cf51ea..eeeef145407 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -826,6 +826,10 @@ "PEM.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", "PET.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", "SMS_D_Ln3.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "ERS.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "PEM.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "PET.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "SMS_D_Ln3.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", ) }, From b1a35ff19682eca0fb2e54528e5778dc49285881 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 22 Oct 2024 10:46:03 -0500 Subject: [PATCH 241/366] trim down wav dev test suite --- cime_config/tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index eeeef145407..a72c4376f0b 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -822,9 +822,6 @@ "e3sm_wav_developer" : { "time" : "0:45:00", "tests" : ( - "ERS.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", - "PEM.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", - "PET.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", "SMS_D_Ln3.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", "ERS.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", "PEM.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", From 0404047801048cae81beb9e7d7d52fcd7c1c64ed Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Mon, 28 Oct 2024 11:06:55 -0500 Subject: [PATCH 242/366] correction to langmuir mixing logic for LandIce Mask. --- .../src/shared/mpas_ocn_vmix_cvmix.F | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F index d1fdc41167c..c059fd5e9ca 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F @@ -142,6 +142,7 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim maxLevelCell, minLevelCell, nEdgesOnCell, maxLevelEdgeTop, minLevelEdgeBot integer, dimension(:), pointer :: landIceMask + integer :: landIceMaskValue real (kind=RKIND), dimension(:), pointer :: & latCell, lonCell, bottomDepth, fCell, & @@ -482,19 +483,25 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim Nsqr_iface(k:maxLevelCell(iCell)+1) = Nsqr_iface(k-1) ! compute Langmuir number and Langmuir enhancement factor - if (config_cvmix_kpp_use_theory_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - langmuirNumber(iCell) = sqrt(surfaceFrictionVelocity(iCell) / ( & + if (associated(landIceMask)) then + landIceMaskValue = landIceMask(iCell) + else + landIceMaskValue = 0 + endif + + if ( landIceMaskValue .eq. 0 .and. iceFraction(iCell) .lt. 0.05_RKIND) then + if (config_cvmix_kpp_use_theory_wave) then + langmuirNumber(iCell) = sqrt(surfaceFrictionVelocity(iCell) / ( & cvmix_kpp_ustokes_SL_model(windSpeed10m(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params)+1e-15_RKIND) ) - langmuirEnhancementFactor = & + langmuirEnhancementFactor = & cvmix_kpp_EFactor_model(windSpeed10m(iCell), & surfaceFrictionVelocity(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params) - else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND & - .and. landIceMask(iCell) .eq. 0) then - call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & + else if (config_cvmix_kpp_use_active_wave) then + call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & windStressMeridional(iCell), & surfaceFrictionVelocity(iCell), & significantWaveHeight(iCell), & @@ -503,9 +510,10 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim stokesDriftMeridionalWavenumber(:,iCell), & alphaAngle, & langmuirNumber(iCell)) - call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & + call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & langmuirNumber(iCell), & langmuirEnhancementFactor) + endif else ! arbitrarily large Langmuir number langmuirNumber(iCell) = 1.e10_RKIND @@ -682,15 +690,21 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim OBL_depth = boundaryLayerDepth(iCell) ) ! update Langmuir enhancement factor - if (config_cvmix_kpp_use_theory_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - langmuirEnhancementFactor = & + if (associated(landIceMask)) then + landIceMaskValue = landIceMask(iCell) + else + landIceMaskValue = 0 + endif + + if (landIceMaskValue .eq. 0 .and. iceFraction(iCell) .lt. 0.05_RKIND) then + if (config_cvmix_kpp_use_theory_wave) then + langmuirEnhancementFactor = & cvmix_kpp_EFactor_model(windSpeed10m(iCell), & surfaceFrictionVelocity(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params) - else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND & - .and. landIceMask(iCell) .eq. 0) then - call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & + else if (config_cvmix_kpp_use_active_wave) then + call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & windStressMeridional(iCell), & surfaceFrictionVelocity(iCell), & significantWaveHeight(iCell), & @@ -699,13 +713,13 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim stokesDriftMeridionalWavenumber(:,iCell), & alphaAngle, & langmuirNumber(iCell)) - call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & - langmuirNumber(iCell), & - langmuirEnhancementFactor) + call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & + langmuirNumber(iCell), & + langmuirEnhancementFactor) + endif else langmuirEnhancementFactor = 1.0_RKIND - end if - + endif ! call mpas_timer_start('cvmix coeffs kpp', .false.) call cvmix_coeffs_kpp( & Mdiff_out = cvmix_variables % Mdiff_iface(minLevelCell(iCell):maxLevelCell(iCell)+1), & From d9ced4307dfc19dcb295f2ad988bc800e4c30eda Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 5 Nov 2024 10:10:51 -0600 Subject: [PATCH 243/366] cleanup whitespace --- driver-mct/cime_config/namelist_definition_drv.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index efb24402e78..da823f1a839 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -4891,7 +4891,7 @@ X - + char mapping From 0a87f8badf9f66a309f76240dea20700c579fdf1 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 5 Nov 2024 10:19:32 -0600 Subject: [PATCH 244/366] remove flux wave to ocean remapping (will be done in later PR) --- driver-mct/main/prep_ocn_mod.F90 | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/driver-mct/main/prep_ocn_mod.F90 b/driver-mct/main/prep_ocn_mod.F90 index 6577f1c6034..605e0019c86 100644 --- a/driver-mct/main/prep_ocn_mod.F90 +++ b/driver-mct/main/prep_ocn_mod.F90 @@ -66,7 +66,6 @@ module prep_ocn_mod public :: prep_ocn_get_mapper_Sg2o public :: prep_ocn_get_mapper_Fg2o public :: prep_ocn_get_mapper_Sw2o - public :: prep_ocn_get_mapper_Fw2o !-------------------------------------------------------------------------- ! Private interfaces @@ -91,7 +90,6 @@ module prep_ocn_mod type(seq_map), pointer :: mapper_Fg2o type(seq_map), pointer :: mapper_Sg2o type(seq_map), pointer :: mapper_Sw2o - type(seq_map), pointer :: mapper_Fw2o ! attribute vectors type(mct_aVect), pointer :: a2x_ox(:) ! Atm export, ocn grid, cpl pes @@ -192,7 +190,6 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc allocate(mapper_Sg2o) allocate(mapper_Fg2o) allocate(mapper_Sw2o) - allocate(mapper_Fw2o) if (ocn_present) then @@ -385,13 +382,6 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc call seq_map_init_rcfile(mapper_Sw2o, wav(1), ocn(1), & 'seq_maps.rc', 'wav2ocn_smapname:', 'wav2ocn_smaptype:',samegrid_ow, & 'mapper_Sw2o initialization') - if (iamroot_CPLID) then - write(logunit,*) ' ' - write(logunit,F00) 'Initializing mapper_Fw2o' - end if - call seq_map_init_rcfile(mapper_Fw2o, wav(1), ocn(1), & - 'seq_maps.rc', 'wav2ocn_fmapname:', 'wav2ocn_fmaptype:',samegrid_ow, & - 'mapper_Fw2o initialization') endif call shr_sys_flush(logunit) @@ -1558,9 +1548,4 @@ function prep_ocn_get_mapper_Sw2o() prep_ocn_get_mapper_Sw2o => mapper_Sw2o end function prep_ocn_get_mapper_Sw2o - function prep_ocn_get_mapper_Fw2o() - type(seq_map), pointer :: prep_ocn_get_mapper_Fw2o - prep_ocn_get_mapper_Fw2o => mapper_Fw2o - end function prep_ocn_get_mapper_Fw2o - end module prep_ocn_mod From 15a544b8ca6e7ab102b1198a2d42b7af4f23bdde Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 5 Nov 2024 10:23:40 -0600 Subject: [PATCH 245/366] remove white space --- driver-mct/main/prep_ocn_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-mct/main/prep_ocn_mod.F90 b/driver-mct/main/prep_ocn_mod.F90 index 605e0019c86..12f36380c83 100644 --- a/driver-mct/main/prep_ocn_mod.F90 +++ b/driver-mct/main/prep_ocn_mod.F90 @@ -1547,5 +1547,5 @@ function prep_ocn_get_mapper_Sw2o() type(seq_map), pointer :: prep_ocn_get_mapper_Sw2o prep_ocn_get_mapper_Sw2o => mapper_Sw2o end function prep_ocn_get_mapper_Sw2o - + end module prep_ocn_mod From 492554f9140ac90704a34391a0b3548ffb0b9120 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 5 Nov 2024 10:39:18 -0600 Subject: [PATCH 246/366] remove WAV2OCN_FMAPNAME --- cime_config/config_grids.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 13fff49e823..9ef7dc51992 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -3655,7 +3655,6 @@ ATM2WAV_SMAPNAME OCN2WAV_SMAPNAME WAV2OCN_SMAPNAME - WAV2OCN_FMAPNAME ICE2WAV_SMAPNAME ROF2OCN_LIQ_RMAPNAME @@ -5076,14 +5075,12 @@ - cpl/gridmaps/wQU225EC30to60E2r2/map_wQU225EC30to60E2r2_TO_EC30to60E2r2_aave.20220222.nc cpl/gridmaps/wQU225EC30to60E2r2/map_wQU225EC30to60E2r2_TO_EC30to60E2r2_blin.20220222.nc cpl/gridmaps/wQU225EC30to60E2r2/map_EC30to60E2r2_TO_wQU225EC30to60E2r2_blin.20220222.nc cpl/gridmaps/wQU225EC30to60E2r2/map_EC30to60E2r2_TO_wQU225EC30to60E2r2_blin.20220222.nc - cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfaave.20240910.nc cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfbilin.20240910.nc cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc From ed5d04fe2797205141dbb69747aa8a5213d01653 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 5 Nov 2024 10:44:51 -0600 Subject: [PATCH 247/366] remove wav to ocean flux remapping --- driver-mct/cime_config/config_component.xml | 17 ----------- .../cime_config/namelist_definition_drv.xml | 30 ------------------- 2 files changed, 47 deletions(-) diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index b917809f6fa..8cddad0abf8 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -2002,23 +2002,6 @@ wav2ocn state mapping file decomp type - - char - idmap - run_domain - env_run.xml - wav2ocn flux mapping file - - - - char - X,Y - X - run_domain - env_run.xml - wav2ocn flux mapping file decomp type - - char idmap diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index da823f1a839..982acabe64d 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -4982,36 +4982,6 @@ - - char - mapping - abs - seq_maps - - wav to ocn flux mapping file for fluxes - - - $WAV2OCN_FMAPNAME - - - - - char - mapping - seq_maps - - The type of mapping desired, either "source" or "destination" mapping. - X is associated with rearrangement of the source grid to the - destination grid and then local mapping. Y is associated with mapping - on the source grid and then rearrangement and sum to the destination - grid. - - - $WAV2OCN_FMAPTYPE - X - - - char(10) drv_physics From abfd3757b68b6e566f67a3dc2e74c24b7c69bca1 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 5 Nov 2024 13:23:49 -0600 Subject: [PATCH 248/366] clean up seq_map_map line for Sw2o --- driver-mct/main/prep_ocn_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-mct/main/prep_ocn_mod.F90 b/driver-mct/main/prep_ocn_mod.F90 index 12f36380c83..b726b1516b2 100644 --- a/driver-mct/main/prep_ocn_mod.F90 +++ b/driver-mct/main/prep_ocn_mod.F90 @@ -1446,7 +1446,7 @@ subroutine prep_ocn_calc_w2x_ox(timer) call t_drvstartf (trim(timer),barrier=mpicom_CPLID) do ewi = 1,num_inst_wav w2x_wx => component_get_c2x_cx(wav(ewi)) - call seq_map_map(mapper_Sw2o, w2x_wx, w2x_ox(ewi),fldlist=seq_flds_w2x_states, norm=.true.) + call seq_map_map(mapper_Sw2o, w2x_wx, w2x_ox(ewi), norm=.true.) enddo call t_drvstopf (trim(timer)) end subroutine prep_ocn_calc_w2x_ox From 9b3ae7880d6b86b1213936029abc5a5bea95ccac Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 6 Nov 2024 13:44:33 -0700 Subject: [PATCH 249/366] Workflows: reworked how to skip eamxx testing jobs * Remove check-skip-labels action * For each eamxx workflow * Run a "pre-process" jobs on ubuntu-latest to - retrieve labels - check touched files Then, add skip logic to other jobs to avoid running unnecessarily if certain labels are found or touched files are irrelevant --- .github/actions/check-skip-labels/README.md | 24 ----- .github/actions/check-skip-labels/action.yml | 48 --------- .github/workflows/eamxx-sa-testing.yml | 102 +++++++++++++------ .github/workflows/eamxx-scripts-tests.yml | 59 ++++++++--- .github/workflows/eamxx-v1-testing.yml | 76 ++++++++++---- 5 files changed, 169 insertions(+), 140 deletions(-) delete mode 100644 .github/actions/check-skip-labels/README.md delete mode 100644 .github/actions/check-skip-labels/action.yml diff --git a/.github/actions/check-skip-labels/README.md b/.github/actions/check-skip-labels/README.md deleted file mode 100644 index 42d1e3bc64e..00000000000 --- a/.github/actions/check-skip-labels/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Composite action to check if we can skip a job for a PR - -This action is meant to be used inside a PR testing workflow, as - -```yaml -jobs: - my-testing: - steps: - ... - - name: check-pr-labels - if: github.event_name == "pull_request" || github.event_name == "pull_request_review" - uses: ./.github/actions/check-skip-labels - with: - skip_labels: label1,label2,label3 -``` - -The input skip_label is a comma-separated list of labels that, if found -on the PR, will cause this job to terminate immediately with a PASS state. - -Ideally, we would like to run this check at the job level, so that we can -skip the job altogether (without using runner time). But while for the -pull_request event we DO have access to the labels from the gh context -(and therefore can check), for pull_request_review we don't, so we need -to ping github for some information diff --git a/.github/actions/check-skip-labels/action.yml b/.github/actions/check-skip-labels/action.yml deleted file mode 100644 index 55d35bfa2b0..00000000000 --- a/.github/actions/check-skip-labels/action.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: 'Check Skip Labels' -description: 'Check for specific skip labels in a pull request' -inputs: - skip_labels: - description: 'Comma-separated list of skip labels' - required: true - token: - description: 'GitHub token for authentication' - required: true - pr_number: - description: 'Pull request number' - required: true - -# Note: inputs are available as env vars in the shell run steps, convertet to uppercase - -runs: - using: "composite" - steps: - - name: Get Pull Request Labels - run: | - echo "Fetching pull request labels..." - if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - LABELS="$SKIP_LABELS" - elif [[ "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then - response=$(curl -s -H "Authorization: token $TOKEN" \ - "https://api.github.com/repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER") - # TODO: reinstante jq once the SNL image is rebuilt! - # LABELS=$(echo "$response" | jq -r '.labels | map(.name) | join(",")') - LABELS=$(echo "$response" | grep -o '"name": *"[^"]*"' | sed 's/"name": *//;s/"//g' | tr '\n' ',' | sed 's/,$//') - fi - echo "labels=$LABELS" >> $GITHUB_ENV - shell: sh - - name: Check for Skip Labels - run: | - echo "Checking for skip labels..." - IFS=',' read -r -a SKIP_LABELS <<< "$SKIP_LABELS" - IFS=',' read -r -a LABEL_ARRAY <<< "$labels" - - for label in "${SKIP_LABELS[@]}"; do - for pr_label in "${LABEL_ARRAY[@]}"; do - if [[ "$pr_label" == "$label" ]]; then - echo "Found skip label '$label'. Skipping this job." - exit 0 # Exit with success status - fi - done - done - echo "No relevant skip labels found. Continuing with the job." - shell: sh diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index 12e45514ad0..0c0bca08413 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -5,17 +5,6 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] - paths: - - components/eamxx/** - - components/eam/src/physics/rrtmgp/** - - components/eam/src/physics/p3/scream/** - - components/eam/src/physics/cam/** - - .github/workflows/eamxx-standalone-testing.yml - - externals/ekat/** - - externals/scorpio/** - - externals/haero/** - - externals/YAKL/** - - components/eam/src/physics/rrtmgp/external/** # Manual run is used to bless workflow_dispatch: @@ -48,15 +37,68 @@ env: submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }} # Submit to cdash only for nightlies jobs: + pre_process_pr: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest # This job can run anywhere + outputs: + relevant_paths: ${{ steps.check_paths.outputs.value }} + labels: ${{ steps.get_labels.outputs.labels }} + steps: + - id: check_paths + run: | + paths=( + components/eamxx + components/eam/src/physics/rrtmgp + components/eam/src/physics/p3/scream + components/eam/src/physics/cam + components/eam/src/physics/rrtmgp/external + externals/ekat + externals/scorpio + externals/haero + externals/YAKL + .github/workflows/eamxx-sa-testing.yml + ) + pattern=$(IFS=\|; echo "${paths[*]}") + + # Use the GitHub API to get the list of changed files + response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}/files") + changed_files=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g') + + # Check for matches and echo the matching files (or "" if none) + matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") + if [[ -n "$matching_files" ]]; then + echo "Found relevant files: $matching_files" + echo "value=true" >> $GITHUB_OUTPUT + else + echo "No relevant files touched by this PR." + echo "value=false" >> $GITHUB_OUTPUT + fi + - id: get_labels + run: | + labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" + echo "labels=${labels}" >> $GITHUB_OUTPUT gcc-openmp: + needs: [pre_process_pr] + if: | + github.event_name == 'schedule' || + ( + github.event_name == 'pull_request' && + needs.pre_process_pr.outputs.relevant_paths=='true' && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip openmp') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') + ) || ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'gcc-openmp' || + github.event.inputs.job_to_run == 'all' + ) runs-on: [self-hosted, ghci-snl-cpu, gcc] strategy: fail-fast: false matrix: build_type: [sp, dbg, fpe, opt] - if: ${{ github.event_name != 'workflow_dispatch' || - github.event.inputs.job_to_run == 'gcc-openmp' || - github.event.inputs.job_to_run == 'all' }} name: gcc-openmp / ${{ matrix.build_type }} steps: - name: Check out the repository @@ -67,13 +109,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip gcc,AT: skip openmp,AT: skip eamxx-sa,AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Set test-all inputs based on event specs run: | echo "generate=false" >> $GITHUB_ENV @@ -91,14 +126,26 @@ jobs: submit: ${{ env.submit }} cmake-configs: Kokkos_ENABLE_OPENMP=ON gcc-cuda: + needs: [pre_process_pr] + if: | + github.event_name == 'schedule' || + ( + github.event_name == 'pull_request' && + needs.pre_process_pr.outputs.relevant_paths=='true' && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip cuda') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') + ) || ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'gcc-cuda' || + github.event.inputs.job_to_run == 'all' + ) runs-on: [self-hosted, ghci-snl-cuda, cuda, gcc] strategy: fail-fast: false matrix: build_type: [sp, dbg, opt] - if: ${{ github.event_name != 'workflow_dispatch' || - github.event.inputs.job_to_run == 'gcc-cuda' || - github.event.inputs.job_to_run == 'all' }} name: gcc-cuda / ${{ matrix.build_type }} steps: - name: Check out the repository @@ -109,13 +156,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip gcc,AT: skip cuda,AT: skip eamxx-sa,AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Set test-all inputs based on event specs run: | echo "generate=false" >> $GITHUB_ENV diff --git a/.github/workflows/eamxx-scripts-tests.yml b/.github/workflows/eamxx-scripts-tests.yml index bd719539bab..d41f3eaf62a 100644 --- a/.github/workflows/eamxx-scripts-tests.yml +++ b/.github/workflows/eamxx-scripts-tests.yml @@ -5,15 +5,6 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] - paths: - - components/eamxx/scripts/** - - components/eamxx/cime_config/*.py - - .github/workflows/eamxx-scripts-tests.yml - - externals/ekat/** - - externals/scorpio/** - - externals/haero/** - - externals/YAKL/** - - components/eam/src/physics/rrtmgp/external/** # Manual run for debug purposes only workflow_dispatch: @@ -30,7 +21,50 @@ concurrency: cancel-in-progress: true jobs: + pre_process_pr: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest # This job can run anywhere + outputs: + relevant_paths: ${{ steps.check_paths.outputs.value}} + labels: ${{ steps.get_labels.outputs.labels }} + steps: + - id: check_paths + run: | + paths=( + components/eamxx/scripts + components/eamxx/cime_config/eamxx + components/eamxx/cime_config/build + components/eamxx/cime_config/yaml_utils.py + .github/workflows/eamxx-scripts-tests.yml + ) + pattern=$(IFS=\|; echo "${paths[*]}") + + # Use the GitHub API to get the list of changed files + response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}/files") + changed_files=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g') + + # Check for matches and echo the matching files (or "" if none) + matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") + if [[ -n "$matching_files" ]]; then + echo "Found relevant files: $matching_files" + echo "value=true" >> $GITHUB_OUTPUT + else + echo "No relevant files touched by this PR." + echo "value=false" >> $GITHUB_OUTPUT + fi + - id: get_labels + run: | + labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" + echo "labels=${labels}" >> $GITHUB_OUTPUT cpu-gcc: + needs: [pre_process_pr] + if: | + github.event_name != 'pull_request' || + ( + needs.pre_process_pr.outputs.relevant_paths == 'true' && + !contains(needs.pre_process_pr.outputs.labels, 'CI: skip eamxx-all') + ) runs-on: [self-hosted, gcc, ghci-snl-cpu] steps: - name: Check out the repository @@ -41,13 +75,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Run test run: | cd components/eamxx diff --git a/.github/workflows/eamxx-v1-testing.yml b/.github/workflows/eamxx-v1-testing.yml index 47129159f82..ca83177fde5 100644 --- a/.github/workflows/eamxx-v1-testing.yml +++ b/.github/workflows/eamxx-v1-testing.yml @@ -5,17 +5,6 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] - paths: - - components/eamxx/** - - components/eam/src/physics/rrtmgp/** - - components/eam/src/physics/p3/scream/** - - components/eam/src/physics/cam/** - - .github/workflows/eamxx-v1-testing.yml - - externals/ekat/** - - externals/scorpio/** - - externals/haero/** - - externals/YAKL/** - - components/eam/src/physics/rrtmgp/external/** # Manual run is used to bless workflow_dispatch: @@ -40,7 +29,62 @@ concurrency: cancel-in-progress: true jobs: + pre_process_pr: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest # This job can run anywhere + outputs: + relevant_paths: ${{ steps.check_paths.outputs.value }} + labels: ${{ steps.get_labels.outputs.labels }} + steps: + - id: check_paths + run: | + paths=( + components/eamxx + components/eam/src/physics/rrtmgp + components/eam/src/physics/p3/scream + components/eam/src/physics/cam + components/eam/src/physics/rrtmgp/external + externals/ekat + externals/scorpio + externals/haero + externals/YAKL + .github/workflows/eamxx-v1-testing.yml + ) + pattern=$(IFS=\|; echo "${paths[*]}") + + # Use the GitHub API to get the list of changed files + response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}/files") + changed_files=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g') + + # Check for matches and echo the matching files (or "" if none) + matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") + if [[ -n "$matching_files" ]]; then + echo "Found relevant files: $matching_files" + echo "value=true" >> $GITHUB_OUTPUT + else + echo "No relevant files touched by this PR." + echo "value=false" >> $GITHUB_OUTPUT + fi + - id: get_labels + run: | + labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" + echo "labels=${labels}" >> $GITHUB_OUTPUT cpu-gcc: + needs: [pre_process_pr] + if: | + github.event_name == 'schedule' || + ( + github.event_name == 'pull_request' && + needs.pre_process_pr.outputs.relevant_paths=='true' && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-v1') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') + ) || ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'cpu-gcc' || + github.event.inputs.job_to_run == 'all' + ) runs-on: [self-hosted, gcc, ghci-snl-cpu] strategy: matrix: @@ -55,9 +99,6 @@ jobs: short_name: SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs fail-fast: false name: cpu-gcc / ${{ matrix.test.short_name }} - if: ${{ github.event_name != 'workflow_dispatch' || - github.event.inputs.job_to_run == 'cpu-gcc' || - github.event.inputs.job_to_run == 'all' }} steps: - name: Check out the repository uses: actions/checkout@v4 @@ -67,13 +108,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip gcc,AT: skip openmp,AT: skip eamxx-sa,AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Set CA certificates env var run: | # Ensure the operating system is Linux From 8f96dfe6c65b1d94ba919e79d17ca9621ef9a0e8 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 5 Nov 2024 11:58:26 -0700 Subject: [PATCH 250/366] ci(Mergify): configuration update Signed-off-by: Luca Bertagna --- .mergify.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .mergify.yml diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 00000000000..81b832dee59 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,31 @@ +merge_protections: + - name: Enforce checks passing + description: Make sure that checks are not failing on the PR, and reviewers approved + if: + - base = master + success_conditions: + - "#approved >= 1" # At least 1 approval + - "#changes-requested == 0" # No reviewer asked for changes + - or: + - and: + - check-success=eamxx-sa/gcc-openmp/sp + - check-success=eamxx-sa/gcc-openmp/dbg + - check-success=eamxx-sa/gcc-openmp/fpe + - check-success=eamxx-sa/gcc-openmp/opt + - check-skipped="eamxx-sa/gcc-openmp/${{matrix.build_type}}" + - or: + - and: + - check-success=eamxx-sa/gcc-cuda/sp + - check-success=eamxx-sa/gcc-cuda/dbg + - check-success=eamxx-sa/gcc-cuda/opt + - check-skipped="eamxx-sa/gcc-cuda/${{matrix.build_type}}" + - or: + - and: + - check-success=eamxx-v1/cpu-gcc/ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2 + - check-success=eamxx-v1/cpu-gcc/ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97 + - check-success=eamxx-v1/cpu-gcc/ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5 + - check-success=eamxx-v1/cpu-gcc/SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs + - check-skipped="eamxx-v1/cpu-gcc-/${{matrix.test.short_name}}" + - or: + - check-success=eamxx-scripts/cpu-gcc + - check-skipped=eamxx-scripts/cpu-gcc From 6f30ec5088c07dcff7b546cd58457efa206cd4c5 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 6 Nov 2024 20:50:58 -0700 Subject: [PATCH 251/366] Mergify: fix syntax --- .mergify.yml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 81b832dee59..85c1375e90f 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -4,28 +4,17 @@ merge_protections: if: - base = master success_conditions: - - "#approved >= 1" # At least 1 approval - - "#changes-requested == 0" # No reviewer asked for changes + - "#approved-reviews-by >= 1" # At least 1 approval + - "#changes-requested-reviews-by == 0" # No reviewer asked for changes - or: - - and: - - check-success=eamxx-sa/gcc-openmp/sp - - check-success=eamxx-sa/gcc-openmp/dbg - - check-success=eamxx-sa/gcc-openmp/fpe - - check-success=eamxx-sa/gcc-openmp/opt - - check-skipped="eamxx-sa/gcc-openmp/${{matrix.build_type}}" + - check-success=eamxx-sa/gcc-openmp/* + - check-skipped=eamxx-sa/gcc-openmp/* - or: - - and: - - check-success=eamxx-sa/gcc-cuda/sp - - check-success=eamxx-sa/gcc-cuda/dbg - - check-success=eamxx-sa/gcc-cuda/opt - - check-skipped="eamxx-sa/gcc-cuda/${{matrix.build_type}}" + - check-success=eamxx-sa/gcc-cuda/* + - check-skipped=eamxx-sa/gcc-cuda/* - or: - - and: - - check-success=eamxx-v1/cpu-gcc/ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2 - - check-success=eamxx-v1/cpu-gcc/ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97 - - check-success=eamxx-v1/cpu-gcc/ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5 - - check-success=eamxx-v1/cpu-gcc/SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs - - check-skipped="eamxx-v1/cpu-gcc-/${{matrix.test.short_name}}" + - check-success=eamxx-v1/cpu-gcc/* + - check-skipped=eamxx-v1/cpu-gcc/* - or: - check-success=eamxx-scripts/cpu-gcc - check-skipped=eamxx-scripts/cpu-gcc From e0a1f20b50e6f9579eba6893c77c8adb90a3f327 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 7 Nov 2024 09:11:57 -0700 Subject: [PATCH 252/366] Remove overload of create_tracer_data_reader. Pass tracer file type to change tag Lev of vertical emissions. --- ...mxx_mam_microphysics_process_interface.cpp | 2 +- .../mam/readfiles/tracer_reader_utils.hpp | 42 +++++++------------ 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 0b1440b3826..6f26646ad6f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -292,7 +292,7 @@ void MAMMicrophysics::set_grids( auto file_reader = scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name, - data_tracer, extfrc_map_file); + data_tracer.file_type); VertEmissionsHorizInterp_.push_back(hor_rem); VertEmissionsDataReader_.push_back(file_reader); vert_emis_data_.push_back(data_tracer); diff --git a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp index e31300a679a..bec628b1f96 100644 --- a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp @@ -72,6 +72,8 @@ enum TracerFileType { ZONAL, // vertical emission files VERT_EMISSION, + // + NONE }; enum TracerDataIndex { BEG = 0, END = 1, OUT = 2 }; @@ -381,11 +383,6 @@ inline std::shared_ptr create_horiz_remapper( model_grid->clone("tracer_horiz_interp_tgt_grid", true); horiz_interp_tgt_grid->reset_num_vertical_lev(tracer_data.nlevs_data); - if(tracer_data.file_type == VERT_EMISSION) { - horiz_interp_tgt_grid->reset_field_tag_name(LEV, "altitude"); - horiz_interp_tgt_grid->reset_field_tag_name(ILEV, "altitude_int"); - } - const int ncols_model = model_grid->get_num_global_dofs(); std::shared_ptr remapper; if(tracer_data.ncols_data == ncols_model) { @@ -436,44 +433,33 @@ inline std::shared_ptr create_horiz_remapper( } // create_horiz_remapper -inline std::shared_ptr create_tracer_data_reader( - const std::shared_ptr &horiz_remapper, - const std::string &tracer_data_file) { - std::vector io_fields; - for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - io_fields.push_back(horiz_remapper->get_src_field(i)); - } - const auto io_grid = horiz_remapper->get_src_grid(); - return std::make_shared(tracer_data_file, io_grid, io_fields, - true); -} // create_tracer_data_reader - inline std::shared_ptr create_tracer_data_reader( const std::shared_ptr &horiz_remapper, const std::string &tracer_data_file, - const TracerData &tracer_data, - const std::string &extfrc_map_file - ) { + const TracerFileType file_type = NONE) +{ std::vector io_fields; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { io_fields.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); - - // NOTE: If we are using a vertical emission NC file with altitude instead of levels, - // we must rename this tag. This is only necessary when a map file is used. - if(tracer_data.file_type == VERT_EMISSION && extfrc_map_file != ""){ + if(file_type == VERT_EMISSION ){ + // NOTE: If we are using a vertical emission nc file with altitude instead of lev, + // we must rename this tag. + // We need to perform a shallow clone of io_grid because tags are const in this object. auto horiz_interp_src_grid = io_grid->clone("tracer_horiz_interp_src_grid", true); - horiz_interp_src_grid->reset_field_tag_name(LEV, "altitude"); - horiz_interp_src_grid->reset_field_tag_name(ILEV, "altitude_int"); - return std::make_shared(tracer_data_file, horiz_interp_src_grid, io_fields, + horiz_interp_src_grid->reset_field_tag_name(LEV, "altitude"); + horiz_interp_src_grid->reset_field_tag_name(ILEV, "altitude_int"); + return std::make_shared(tracer_data_file, horiz_interp_src_grid, io_fields, true); - } else { + } else{ + // We do not need to rename tags in or clone io_grid for other types of files. return std::make_shared(tracer_data_file, io_grid, io_fields, true); } + } // create_tracer_data_reader inline void update_tracer_data_from_file( From deb0ceba431b798545f446466dbf46fe5dbe9610 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 7 Nov 2024 11:41:27 -0700 Subject: [PATCH 253/366] Workflows: nano-fixes to eamxx workflows * Add name entry for pre_process_pr steps * Ensure actual jobs run if pre_process_pr is skipped --- .github/workflows/eamxx-sa-testing.yml | 10 ++++++---- .github/workflows/eamxx-scripts-tests.yml | 6 ++++-- .github/workflows/eamxx-v1-testing.yml | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index 0c0bca08413..bbfa12b80ac 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -44,7 +44,8 @@ jobs: relevant_paths: ${{ steps.check_paths.outputs.value }} labels: ${{ steps.get_labels.outputs.labels }} steps: - - id: check_paths + - name: Check files modified by PR + id: check_paths run: | paths=( components/eamxx @@ -74,14 +75,15 @@ jobs: echo "No relevant files touched by this PR." echo "value=false" >> $GITHUB_OUTPUT fi - - id: get_labels + - name: Retrieve PR labels + id: get_labels run: | labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" echo "labels=${labels}" >> $GITHUB_OUTPUT gcc-openmp: needs: [pre_process_pr] if: | - github.event_name == 'schedule' || + success() && github.event_name == 'schedule' || ( github.event_name == 'pull_request' && needs.pre_process_pr.outputs.relevant_paths=='true' && @@ -128,7 +130,7 @@ jobs: gcc-cuda: needs: [pre_process_pr] if: | - github.event_name == 'schedule' || + success() && github.event_name == 'schedule' || ( github.event_name == 'pull_request' && needs.pre_process_pr.outputs.relevant_paths=='true' && diff --git a/.github/workflows/eamxx-scripts-tests.yml b/.github/workflows/eamxx-scripts-tests.yml index d41f3eaf62a..6d39200e273 100644 --- a/.github/workflows/eamxx-scripts-tests.yml +++ b/.github/workflows/eamxx-scripts-tests.yml @@ -28,7 +28,8 @@ jobs: relevant_paths: ${{ steps.check_paths.outputs.value}} labels: ${{ steps.get_labels.outputs.labels }} steps: - - id: check_paths + - name: Check files modified by PR + id: check_paths run: | paths=( components/eamxx/scripts @@ -53,7 +54,8 @@ jobs: echo "No relevant files touched by this PR." echo "value=false" >> $GITHUB_OUTPUT fi - - id: get_labels + - name: Retrieve PR labels + id: get_labels run: | labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" echo "labels=${labels}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/eamxx-v1-testing.yml b/.github/workflows/eamxx-v1-testing.yml index ca83177fde5..3a4b28cfed3 100644 --- a/.github/workflows/eamxx-v1-testing.yml +++ b/.github/workflows/eamxx-v1-testing.yml @@ -36,7 +36,8 @@ jobs: relevant_paths: ${{ steps.check_paths.outputs.value }} labels: ${{ steps.get_labels.outputs.labels }} steps: - - id: check_paths + - name: Check files modified by PR + id: check_paths run: | paths=( components/eamxx @@ -66,7 +67,8 @@ jobs: echo "No relevant files touched by this PR." echo "value=false" >> $GITHUB_OUTPUT fi - - id: get_labels + - name: Retrieve PR labels + id: get_labels run: | labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" echo "labels=${labels}" >> $GITHUB_OUTPUT From d9691e6043c85fdf51a380eaa1e7b9939f994573 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Tue, 5 Nov 2024 15:47:32 -0600 Subject: [PATCH 254/366] Updating to SCORPIO v1.6.6 Updating SCORPIO version from 1.6.5 to 1.6.6 SCORPIO v1.6.6 includes the following fixes/enhancements, * Support for apps that use I/O type PIO_IOTYPE_ADIOSC (Build support only, ADIOS BP output with compression is not supported yet) --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index 804f87c48fc..59601f625e3 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit 804f87c48fc9013009307b806d1aff2afa3a0d7c +Subproject commit 59601f625e34019de0e1d3411d07bdc9e70d0054 From 776a79a9083f592b93a0f469899aca4c63f6abf1 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 7 Nov 2024 12:20:07 -0700 Subject: [PATCH 255/366] Update .mergify.yml Disable comments from merge_protections on PRs --- .mergify.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.mergify.yml b/.mergify.yml index 85c1375e90f..e931d54eb09 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,6 +1,7 @@ merge_protections: - name: Enforce checks passing description: Make sure that checks are not failing on the PR, and reviewers approved + comment: false if: - base = master success_conditions: From 57e02f082f99886d364555268d760d22bd0b7039 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Thu, 7 Nov 2024 13:38:23 -0600 Subject: [PATCH 256/366] increase time for wav tests and increase processor count for PEM test --- cime_config/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index a72c4376f0b..d522aebe36c 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -820,11 +820,11 @@ }, "e3sm_wav_developer" : { - "time" : "0:45:00", + "time" : "1:00:00", "tests" : ( "SMS_D_Ln3.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", "ERS.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", - "PEM.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "PEM_P480.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", "PET.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", "SMS_D_Ln3.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", ) From b87fd0b56893b0af13f6629e94572ecaafa5fdac Mon Sep 17 00:00:00 2001 From: Donghui Xu Date: Thu, 7 Nov 2024 14:33:06 -0800 Subject: [PATCH 257/366] change the coupling period of MOSART to be half hour in land river two-way coupling to avoid time step mismatch --- .../testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands | 2 ++ .../testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands new file mode 100644 index 00000000000..6eeec0102e5 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands @@ -0,0 +1,2 @@ +./xmlchange LND_NCPL=48 +./xmlchange ROF_NCPL=48 \ No newline at end of file diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart index c7a09f76ae4..fe460362de6 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart @@ -1 +1,2 @@ -inundflag = .true. \ No newline at end of file +inundflag = .true. +delt_mosart = 1800 \ No newline at end of file From 2b6fd04d4b484fa28d41ef750444e2068407c9a4 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 7 Nov 2024 15:45:25 -0700 Subject: [PATCH 258/366] pre sed --- .../{ => infra}/p3_unit_tests_common.hpp | 35 ++++++++++++++++++- .../p3/tests/p3_autoconversion_unit_tests.cpp | 3 -- .../p3_back_to_cell_average_unit_tests.cpp | 4 --- ...lc_liq_relaxation_timescale_unit_tests.cpp | 4 --- .../tests/p3_calc_rime_density_unit_tests.cpp | 3 -- .../p3/tests/p3_check_values_unit_tests.cpp | 1 - .../p3_cldliq_imm_freezing_unit_tests.cpp | 3 -- .../p3/tests/p3_cloud_rain_acc_unit_tests.cpp | 3 -- .../p3/tests/p3_cloud_sed_unit_tests.cpp | 4 --- .../tests/p3_droplet_self_coll_unit_tests.cpp | 3 -- .../physics/p3/tests/p3_dsd2_unit_tests.cpp | 6 ---- .../p3/tests/p3_evaporate_rain_unit_tests.cpp | 3 -- .../p3_ice_cldliq_wet_growth_unit_tests.cpp | 3 -- .../p3/tests/p3_ice_collection_unit_tests.cpp | 9 ----- .../p3_ice_deposition_sublimation_tests.cpp | 3 -- .../p3/tests/p3_ice_melting_unit_tests.cpp | 3 -- .../p3/tests/p3_ice_nucleation_unit_tests.cpp | 3 -- ...p3_ice_relaxation_timescale_unit_tests.cpp | 3 -- .../p3/tests/p3_ice_sed_unit_tests.cpp | 10 ------ .../p3_ice_supersat_conservation_tests.cpp | 4 --- .../p3/tests/p3_ice_tables_unit_tests.cpp | 3 -- .../p3_incloud_mixingratios_unit_tests.cpp | 3 -- .../physics/p3/tests/p3_main_unit_tests.cpp | 13 ------- .../p3/tests/p3_nc_conservation_tests.cpp | 4 --- .../p3/tests/p3_ni_conservation_tests.cpp | 4 --- .../p3/tests/p3_nr_conservation_tests.cpp | 4 --- .../p3_prevent_liq_supersaturation_tests.cpp | 4 --- .../tests/p3_rain_imm_freezing_unit_tests.cpp | 3 -- .../p3/tests/p3_rain_sed_unit_tests.cpp | 17 +++------ .../tests/p3_rain_self_collection_tests.cpp | 3 -- ...p3_subgrid_variance_scaling_unit_tests.cpp | 14 ++------ .../src/physics/p3/tests/p3_unit_tests.cpp | 21 ----------- .../physics/p3/tests/p3_upwind_unit_tests.cpp | 20 ++++------- .../share/util/scream_setup_random_test.hpp | 8 +++-- 34 files changed, 53 insertions(+), 178 deletions(-) rename components/eamxx/src/physics/p3/tests/{ => infra}/p3_unit_tests_common.hpp (82%) diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp similarity index 82% rename from components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp index 36614cf9a88..523315ae5ec 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp @@ -2,6 +2,7 @@ #define P3_UNIT_TESTS_COMMON_HPP #include "share/scream_types.hpp" +#include "share/util/scream_setup_random_test.hpp" #include "p3_functions.hpp" #include "p3_data.hpp" #include "ekat/util/ekat_test_utils.hpp" @@ -72,11 +73,15 @@ struct UnitWrap { struct Base { std::string m_baseline_path; + std::string m_test_name; BASELINE_ACTION m_baseline_action; + ekat::FILEPtr m_fid; Base() : m_baseline_path(""), - m_baseline_action(NONE) + m_test_name(Catch::getResultCapture().getCurrentTestName()), + m_baseline_action(NONE), + m_fid() { Functions::p3_init(); // many tests will need fortran table data auto& ts = ekat::TestSession::get(); @@ -104,12 +109,40 @@ struct UnitWrap { } EKAT_REQUIRE_MSG( !(m_baseline_action != NONE && m_baseline_path == ""), "P3 unit test flags problem: baseline actions were requested but no baseline path was provided"); + + std::string baseline_name = m_baseline_path + "/" + m_test_name; + if (m_baseline_action == COMPARE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + } + else if (m_baseline_action == GENERATE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + } } ~Base() { scream::p3::P3GlobalForFortran::deinit(); } + + std::mt19937_64 get_engine() + { + if (m_baseline_action != COMPARE) { + // We can use any seed + int seed; + auto engine = setup_random_test(nullptr, &seed); + if (m_baseline_action == GENERATE) { + // Write the seed + ekat::write(&seed, 1, m_fid); + } + return engine; + } + else { + // Read the seed + int seed; + ekat::read(&seed, 1, m_fid); + return setup_random_test(seed); + } + } }; // Put struct decls here diff --git a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp index cb799a67287..bc99ac99150 100644 --- a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp @@ -60,9 +60,7 @@ void cloud_water_autoconversion_unit_bfb_tests() { Kokkos::deep_copy(cwadc_device, cwadc_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/cloud_water_autoconversion.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { cwadc[i].read(fid); } @@ -117,7 +115,6 @@ void cloud_water_autoconversion_unit_bfb_tests() { } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cwadc_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index aa70e7fe42c..1e36a024508 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -5,7 +5,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -45,9 +44,7 @@ void run_bfb() Kokkos::deep_copy(device_data, host_data); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/back_to_cell_average.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { back_to_cell_average_data[i].read(fid); } @@ -176,7 +173,6 @@ void run_bfb() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { host_data(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index 7f3d7ee126f..e528d27328d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -5,7 +5,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -55,9 +54,7 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: } // Read baseline data - std::string baseline_name = this->m_baseline_path + "/calc_liq_relaxation_timescale.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { self[i].read(fid); } @@ -108,7 +105,6 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: } } else { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { self_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp index ef4cd621843..bb186ac4272 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp @@ -88,9 +88,7 @@ void run_bfb() Kokkos::deep_copy(device_data, host_data); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/calc_rime_density.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { calc_rime_density_data[i].read(fid); } @@ -137,7 +135,6 @@ void run_bfb() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { host_data(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp index 2a858f86dfc..90c8d28be93 100644 --- a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp @@ -6,7 +6,6 @@ #include "p3_functions.hpp" #include "p3_test_data.hpp" #include "p3_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp index 5dacc132a87..06c725d10a8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp @@ -71,9 +71,7 @@ void run_bfb() Kokkos::deep_copy(device_data, host_data); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/cldliq_imm_freezing.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { cldliq_imm_freezing_data[i].read(fid); } @@ -120,7 +118,6 @@ void run_bfb() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { host_data(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp index fff6784ab61..011142942a7 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp @@ -74,9 +74,7 @@ void run_bfb() Kokkos::deep_copy(device_data, host_data); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/cloud_rain_accretion.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { cloud_rain_acc_data[i].read(fid); } @@ -123,7 +121,6 @@ void run_bfb() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { host_data(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index 4f729d8fde6..0c60420f3d8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -5,7 +5,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -57,9 +56,7 @@ void run_bfb() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/cloud_sedimentation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (auto& d : csds_baseline) { d.read(fid); } @@ -91,7 +88,6 @@ void run_bfb() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { csds_cxx[i].write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp index b17d8309c3c..a649d31a489 100644 --- a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp @@ -73,9 +73,7 @@ void run_bfb() Kokkos::deep_copy(device_data, host_data); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/droplet_self_collection.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { droplet_self_coll_data[i].read(fid); } @@ -117,7 +115,6 @@ void run_bfb() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { host_data(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 5bf199fa85a..93187632ced 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -61,9 +61,7 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { Kokkos::deep_copy(gcdd_device, gcdd_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/get_cloud_dsd2.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { gcdd[i].read(fid); } @@ -110,7 +108,6 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { gcdd_host(s).write(fid); } @@ -155,9 +152,7 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { Kokkos::deep_copy(grdd_device, grdd_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/get_rain_dsd2.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { grdd[i].read(fid); } @@ -203,7 +198,6 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { grdd_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index 15efcd85fca..005d13bda2d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -174,9 +174,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest: Kokkos::deep_copy(espd_device, espd_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/evaporate_rain.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { espd[i].read(fid); } @@ -252,7 +250,6 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { espd_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index 3389b52efc6..26f5723ff1c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -58,9 +58,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest Kokkos::deep_copy(self_device, self_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_cldliq_wet_growth.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { self[i].read(fid); } @@ -130,7 +128,6 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { self_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index a2f3e7a8b72..a6e65426dbe 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -64,9 +64,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: Kokkos::deep_copy(cldliq_device, cldliq_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_cldliq_collection.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { cldliq[i].read(fid); } @@ -122,7 +120,6 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cldliq_host(s).write(fid); } @@ -168,9 +165,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: Kokkos::deep_copy(rain_device, rain_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_rain_collection.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { rain[i].read(fid); } @@ -219,7 +214,6 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { rain_host(s).write(fid); } @@ -265,9 +259,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: Kokkos::deep_copy(self_device, self_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_self_collection.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { self[i].read(fid); } @@ -306,7 +298,6 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { self_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp index e5983fb8c74..2ef5e66803a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp @@ -82,9 +82,7 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un Kokkos::deep_copy(cxx_device, cxx_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_deposition_sublimation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { baseline_data[i].read(fid); } @@ -152,7 +150,6 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cxx_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index f3e57a415bd..739f2a31629 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -58,9 +58,7 @@ void ice_melting_bfb() { Kokkos::deep_copy(IceMelt_device, IceMelt_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_melting.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { IceMelt[i].read(fid); } @@ -110,7 +108,6 @@ void ice_melting_bfb() { } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { IceMelt_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp index 47115afd430..f0b4cc4a324 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp @@ -56,9 +56,7 @@ struct UnitWrap::UnitTest::TestIceNucleation : public UnitWrap::UnitTest:: std::string root_name = "ice_nucleation"; std::string file_name = root_name + (do_predict_nc ? "1" : "0") + (do_prescribed_CCN ? "1" : "0"); - std::string baseline_name = this->m_baseline_path + "/" + file_name + ".dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { self[i].read(fid); } @@ -106,7 +104,6 @@ struct UnitWrap::UnitTest::TestIceNucleation : public UnitWrap::UnitTest:: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { self_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp index 08bded63ff4..d0a9b3902f9 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp @@ -50,9 +50,7 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale : public UnitWrap::Unit }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_relaxation_timescale.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { self[i].read(fid); } @@ -104,7 +102,6 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale : public UnitWrap::Unit } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { self_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index c250a15a6ff..6f567a8ae64 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -5,7 +5,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -79,9 +78,7 @@ void run_bfb_calc_bulk_rhime() Kokkos::deep_copy(cbrr_device, cbrr_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/calc_bulk_rho_rime.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { cbrr_baseline[i].read(fid); } @@ -124,7 +121,6 @@ void run_bfb_calc_bulk_rhime() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cbrr_host(s).write(fid); } @@ -160,9 +156,7 @@ void run_bfb_ice_sed() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_sedimentation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < num_runs; ++i) { isds_baseline[i].read(fid); } @@ -198,7 +192,6 @@ void run_bfb_ice_sed() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { isds_cxx[i].write(fid); } @@ -243,9 +236,7 @@ void run_bfb_homogeneous_freezing() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/homogeneous_freezing.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (auto& d : hfds_baseline) { d.read(fid); } @@ -277,7 +268,6 @@ void run_bfb_homogeneous_freezing() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { hfds_cxx[i].write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp index 0e56fb75bec..0324bbf1498 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp @@ -5,7 +5,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -44,9 +43,7 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::Uni Kokkos::deep_copy(cxx_device, cxx_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_supersat_conservation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { baseline_data[i].read(fid); } @@ -90,7 +87,6 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::Uni } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cxx_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp index 92097e57191..22ac35475fb 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp @@ -210,9 +210,7 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base Kokkos::deep_copy(lidb_device, lidb_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/p3_ice_tables_all.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { lid[i].read(fid); lidb[i].read(fid); @@ -296,7 +294,6 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { lid[s].dumi = int_results_mirror(0, s); lid[s].dumjj = int_results_mirror(1, s); diff --git a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp index 2741992b90e..79bffe1ac99 100644 --- a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp @@ -70,9 +70,7 @@ struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest:: Kokkos::deep_copy(self_device, self_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/calculate_incloud_mixingratios.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { self[i].read(fid); } @@ -132,7 +130,6 @@ struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest:: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { self_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index 136fdb2fbe2..26f075dfef3 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -5,7 +5,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -96,9 +95,7 @@ void run_bfb_p3_main_part1() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/p3_main_part1.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (auto& d : isds_baseline) { d.read(fid); } @@ -153,7 +150,6 @@ void run_bfb_p3_main_part1() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { isds_cxx[i].write(fid); } @@ -208,9 +204,7 @@ void run_bfb_p3_main_part2() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/p3_main_part2.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (auto& d : isds_baseline) { d.read(fid); } @@ -287,7 +281,6 @@ void run_bfb_p3_main_part2() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { isds_cxx[i].write(fid); } @@ -335,9 +328,7 @@ void run_bfb_p3_main_part3() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/p3_main_part3.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (auto& d : isds_baseline) { d.read(fid); } @@ -392,7 +383,6 @@ void run_bfb_p3_main_part3() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { isds_cxx[i].write(fid); } @@ -447,9 +437,7 @@ void run_bfb_p3_main() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/p3_main.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (auto& d : isds_baseline) { d.read(fid); } @@ -509,7 +497,6 @@ void run_bfb_p3_main() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { isds_cxx[i].write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp index 26b9beecc7e..288a52c4ad8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp @@ -5,7 +5,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -37,9 +36,7 @@ struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest: Kokkos::deep_copy(cxx_device, cxx_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/nc_conservation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { baseline_data[i].read(fid); } @@ -86,7 +83,6 @@ struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cxx_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp index 6c50154c046..a721dc7b9af 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp @@ -5,7 +5,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -37,9 +36,7 @@ struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest: Kokkos::deep_copy(cxx_device, cxx_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ni_conservation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { baseline_data[i].read(fid); } @@ -85,7 +82,6 @@ struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cxx_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp index b33dd02b0f0..e7a75821ae1 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp @@ -5,7 +5,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -38,9 +37,7 @@ struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest: Kokkos::deep_copy(cxx_device, cxx_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/nr_conservation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { baseline_data[i].read(fid); } @@ -90,7 +87,6 @@ struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cxx_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index 31d18572f3b..803d353eea2 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -4,7 +4,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" #include "p3_test_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "share/scream_types.hpp" #include "physics/share/physics_functions.hpp" @@ -124,9 +123,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U } // Read baseline data - std::string baseline_name = this->m_baseline_path + "/prevent_liq_supersaturation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { baseline_data[i].read(fid); } @@ -180,7 +177,6 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cxx_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp index 875b47e4722..a391a863c72 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp @@ -70,9 +70,7 @@ void run_bfb() Kokkos::deep_copy(device_data, host_data); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/rain_immersion_freezing.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { rain_imm_freezing_data[i].read(fid); } @@ -117,7 +115,6 @@ void run_bfb() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { host_data(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index eac5c4827b6..cd4d1955aed 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -6,7 +6,6 @@ #include "p3_functions.hpp" #include "p3_test_data.hpp" #include "p3_data.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "p3_unit_tests_common.hpp" @@ -78,11 +77,9 @@ void run_bfb_rain_vel() Kokkos::deep_copy(crfv_device, crfv_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/rain_fall_velocity.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { - crfv_baseline[i].read(fid); + crfv_baseline[i].read(Base::m_fid); } } @@ -127,9 +124,8 @@ void run_bfb_rain_vel() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { - crfv_host(s).write(fid); + crfv_host(s).write(Base::m_fid); } } } @@ -137,7 +133,7 @@ void run_bfb_rain_vel() void run_bfb_rain_sed() { // With stored baselines, we must use a fixed seed! - auto engine = setup_random_test(1267351); + auto engine = Base::get_engine(); // F90 is quite slow on weaver, so we decrease dt to reduce // the number of steps in rain_sed. @@ -172,11 +168,9 @@ void run_bfb_rain_sed() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/rain_sed.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (auto& d : rsds_baseline) { - d.read(fid); + d.read(Base::m_fid); } } @@ -215,9 +209,8 @@ void run_bfb_rain_sed() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { - rsds_cxx[i].write(fid); + rsds_cxx[i].write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp index c175f691547..52ff887168b 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp @@ -57,9 +57,7 @@ struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest Kokkos::deep_copy(dc_device, dc_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/rain_self_collection.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { dc[i].read(fid); } @@ -104,7 +102,6 @@ struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { dc_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp index bf1c3543dd2..c58ac68b4a6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp @@ -42,23 +42,13 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::Un view_1d scaling_device("c scaling",1); auto scaling_host = Kokkos::create_mirror_view(scaling_device); - std::string baseline_name = this->m_baseline_path + "/subgrid_variance_scaling.dat"; - ekat::FILEPtr rfid = nullptr; - ekat::FILEPtr wfid = nullptr; - if (this->m_baseline_action == COMPARE) { - rfid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); - } - else if (this->m_baseline_action == GENERATE) { - wfid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); - } - for (Int i = 0; i < 3; ++i) { // loop over exponents for (Int j = 0; j < 16; ++j) { // loop over relvars // Get baseline solution // ---------------------------------- if (this->m_baseline_action == COMPARE) { - ekat::read(&baseline_scaling, 1, rfid); + ekat::read(&baseline_scaling, 1, Base::m_fid); } // Get C++ solution @@ -84,7 +74,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::Un REQUIRE(baseline_scaling == scaling_host(0) ); } else if (this->m_baseline_action == GENERATE) { - ekat::write(&scaling_host(0), 1, wfid); + ekat::write(&scaling_host(0), 1, Base::m_fid); } } //end loop over relvar[j] } //end loop over expons[i] diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index c3f8571d117..5aed6dfe689 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -223,9 +223,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: Kokkos::deep_copy(cwdc_device, cwdc_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/cloud_water_conservation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { cwdc[i].read(fid); } @@ -280,7 +278,6 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { cwdc_host(s).write(fid); } @@ -323,9 +320,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: Kokkos::deep_copy(iwdc_device, iwdc_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/ice_water_conservation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { iwdc[i].read(fid); } @@ -385,7 +380,6 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { iwdc_host(s).write(fid); } @@ -428,9 +422,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: Kokkos::deep_copy(rwdc_device, rwdc_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/rain_water_conservation.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { rwdc[i].read(fid); } @@ -485,7 +477,6 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { rwdc_host(s).write(fid); } @@ -622,9 +613,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitT Kokkos::deep_copy(pupidc_device, pupidc_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/update_prognostic_ice.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { pupidc[i].read(fid); } @@ -728,7 +717,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitT } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { pupidc_host(s).write(fid); } @@ -778,9 +766,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::U Kokkos::deep_copy(gtspvd_device, gtspvd_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/get_time_space_phys_variables.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { gtspvd[i].read(fid); } @@ -852,7 +838,6 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::U } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { gtspvd_host(s).write(fid); } @@ -947,9 +932,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitT Kokkos::deep_copy(pupldc_device, pupldc_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/update_prognostic_liquid.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { pupldc[i].read(fid); } @@ -1036,7 +1019,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitT } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { pupldc_host(s).write(fid); } @@ -1087,9 +1069,7 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi : public UnitWrap: Kokkos::deep_copy(dc_device, dc_host); // Read baseline data - std::string baseline_name = this->m_baseline_path + "/impose_max_total_ni.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (Int i = 0; i < max_pack_size; ++i) { dc[i].read(fid); } @@ -1125,7 +1105,6 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi : public UnitWrap: } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int s = 0; s < max_pack_size; ++s) { dc_host(s).write(fid); } diff --git a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp index 1f65984d727..6e83a8ee4ea 100644 --- a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp @@ -6,8 +6,6 @@ #include "p3_test_data.hpp" #include "share/scream_types.hpp" -#include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" @@ -204,7 +202,7 @@ void run_phys() void run_bfb() { // With stored baselines, we must use a fixed seed! - auto engine = setup_random_test(12345745); + auto engine = Base::get_engine(); CalcUpwindData cuds_baseline[] = { // kts, kte, kdir, kbot, k_qxtop, na, dt_sub, @@ -237,11 +235,9 @@ void run_bfb() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/upwind.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (auto& d : cuds_baseline) { - d.read(fid); + d.read(Base::m_fid); } } @@ -275,9 +271,8 @@ void run_bfb() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { - cuds_cxx[i].write(fid); + cuds_cxx[i].write(Base::m_fid); } } } @@ -295,7 +290,7 @@ void run_phys() void run_bfb() { // With stored baselines, we must use a fixed seed! - auto engine = setup_random_test(2346563); + auto engine = Base::get_engine(); GenSedData gsds_baseline[] = { // kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, num_arrays @@ -322,11 +317,9 @@ void run_bfb() }; // Read baseline data - std::string baseline_name = this->m_baseline_path + "/gen_sed.dat"; if (this->m_baseline_action == COMPARE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); for (auto& d : gsds_baseline) { - d.read(fid); + d.read(Base::m_fid); } } @@ -363,9 +356,8 @@ void run_bfb() } } else if (this->m_baseline_action == GENERATE) { - auto fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); for (Int i = 0; i < num_runs; ++i) { - gsds_cxx[i].write(fid); + gsds_cxx[i].write(Base::m_fid); } } } diff --git a/components/eamxx/src/share/util/scream_setup_random_test.hpp b/components/eamxx/src/share/util/scream_setup_random_test.hpp index 419bd4ee64e..09f3f3099d8 100644 --- a/components/eamxx/src/share/util/scream_setup_random_test.hpp +++ b/components/eamxx/src/share/util/scream_setup_random_test.hpp @@ -41,9 +41,13 @@ inline int get_random_test_seed(const ekat::Comm* comm=nullptr) } template -Engine setup_random_test(const ekat::Comm* comm=nullptr) +Engine setup_random_test(const ekat::Comm* comm=nullptr, int* return_seed=nullptr) { - return Engine (get_random_test_seed(comm)); + int seed = get_random_test_seed(comm); + if (return_seed != nullptr) { + *return_seed = seed; + } + return Engine (seed); } template From 68edf0cff83222d151209d622e4812902ee23b69 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 7 Nov 2024 15:57:00 -0700 Subject: [PATCH 259/366] seds --- .../p3/tests/p3_autoconversion_unit_tests.cpp | 4 +-- .../p3_back_to_cell_average_unit_tests.cpp | 4 +-- ...lc_liq_relaxation_timescale_unit_tests.cpp | 4 +-- .../tests/p3_calc_rime_density_unit_tests.cpp | 4 +-- .../p3_cldliq_imm_freezing_unit_tests.cpp | 4 +-- .../p3/tests/p3_cloud_rain_acc_unit_tests.cpp | 4 +-- .../p3/tests/p3_cloud_sed_unit_tests.cpp | 4 +-- .../tests/p3_droplet_self_coll_unit_tests.cpp | 4 +-- .../physics/p3/tests/p3_dsd2_unit_tests.cpp | 8 +++--- .../p3/tests/p3_evaporate_rain_unit_tests.cpp | 4 +-- .../p3_ice_cldliq_wet_growth_unit_tests.cpp | 4 +-- .../p3/tests/p3_ice_collection_unit_tests.cpp | 12 ++++---- .../p3_ice_deposition_sublimation_tests.cpp | 4 +-- .../p3/tests/p3_ice_melting_unit_tests.cpp | 4 +-- .../p3/tests/p3_ice_nucleation_unit_tests.cpp | 4 +-- ...p3_ice_relaxation_timescale_unit_tests.cpp | 4 +-- .../p3/tests/p3_ice_sed_unit_tests.cpp | 12 ++++---- .../p3_ice_supersat_conservation_tests.cpp | 4 +-- .../p3/tests/p3_ice_tables_unit_tests.cpp | 16 +++++------ .../p3_incloud_mixingratios_unit_tests.cpp | 4 +-- .../physics/p3/tests/p3_main_unit_tests.cpp | 16 +++++------ .../p3/tests/p3_nc_conservation_tests.cpp | 4 +-- .../p3/tests/p3_ni_conservation_tests.cpp | 4 +-- .../p3/tests/p3_nr_conservation_tests.cpp | 4 +-- .../p3_prevent_liq_supersaturation_tests.cpp | 4 +-- .../tests/p3_rain_imm_freezing_unit_tests.cpp | 4 +-- .../tests/p3_rain_self_collection_tests.cpp | 4 +-- .../src/physics/p3/tests/p3_unit_tests.cpp | 28 +++++++++---------- 28 files changed, 90 insertions(+), 90 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp index bc99ac99150..be08e5b0d4f 100644 --- a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp @@ -62,7 +62,7 @@ void cloud_water_autoconversion_unit_bfb_tests() { // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - cwadc[i].read(fid); + cwadc[i].read(Base::m_fid); } } @@ -116,7 +116,7 @@ void cloud_water_autoconversion_unit_bfb_tests() { } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cwadc_host(s).write(fid); + cwadc_host(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index 1e36a024508..f13a0ca909a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -46,7 +46,7 @@ void run_bfb() // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - back_to_cell_average_data[i].read(fid); + back_to_cell_average_data[i].read(Base::m_fid); } } @@ -174,7 +174,7 @@ void run_bfb() } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - host_data(s).write(fid); + host_data(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index e528d27328d..56c2beafaed 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -56,7 +56,7 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - self[i].read(fid); + self[i].read(Base::m_fid); } } @@ -106,7 +106,7 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: } else { for (Int s = 0; s < max_pack_size; ++s) { - self_host(s).write(fid); + self_host(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp index bb186ac4272..9b7722895fa 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp @@ -90,7 +90,7 @@ void run_bfb() // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - calc_rime_density_data[i].read(fid); + calc_rime_density_data[i].read(Base::m_fid); } } @@ -136,7 +136,7 @@ void run_bfb() } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - host_data(s).write(fid); + host_data(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp index 06c725d10a8..bbaa95e87a0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp @@ -73,7 +73,7 @@ void run_bfb() // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - cldliq_imm_freezing_data[i].read(fid); + cldliq_imm_freezing_data[i].read(Base::m_fid); } } @@ -119,7 +119,7 @@ void run_bfb() } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - host_data(s).write(fid); + host_data(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp index 011142942a7..271f7e27bd6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp @@ -76,7 +76,7 @@ void run_bfb() // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - cloud_rain_acc_data[i].read(fid); + cloud_rain_acc_data[i].read(Base::m_fid); } } @@ -122,7 +122,7 @@ void run_bfb() } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - host_data(s).write(fid); + host_data(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index 0c60420f3d8..db42d68e6bf 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -58,7 +58,7 @@ void run_bfb() // Read baseline data if (this->m_baseline_action == COMPARE) { for (auto& d : csds_baseline) { - d.read(fid); + d.read(Base::m_fid); } } @@ -89,7 +89,7 @@ void run_bfb() } else if (this->m_baseline_action == GENERATE) { for (Int i = 0; i < num_runs; ++i) { - csds_cxx[i].write(fid); + csds_cxx[i].write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp index a649d31a489..f183bba65da 100644 --- a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp @@ -75,7 +75,7 @@ void run_bfb() // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - droplet_self_coll_data[i].read(fid); + droplet_self_coll_data[i].read(Base::m_fid); } } @@ -116,7 +116,7 @@ void run_bfb() } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - host_data(s).write(fid); + host_data(s).write(Base::m_fid); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 93187632ced..268607645f0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -63,7 +63,7 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - gcdd[i].read(fid); + gcdd[i].read(Base::m_fid); } } @@ -109,7 +109,7 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - gcdd_host(s).write(fid); + gcdd_host(s).write(Base::m_fid); } } } @@ -154,7 +154,7 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - grdd[i].read(fid); + grdd[i].read(Base::m_fid); } } @@ -199,7 +199,7 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - grdd_host(s).write(fid); + grdd_host(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index 005d13bda2d..a84b1e0448a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -176,7 +176,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - espd[i].read(fid); + espd[i].read(Base::m_fid); } } @@ -251,7 +251,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - espd_host(s).write(fid); + espd_host(s).write(Base::m_fid); } } } // end run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index 26f5723ff1c..cca204b9b3a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -60,7 +60,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - self[i].read(fid); + self[i].read(Base::m_fid); } } @@ -129,7 +129,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - self_host(s).write(fid); + self_host(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index a6e65426dbe..2561114b519 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -66,7 +66,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - cldliq[i].read(fid); + cldliq[i].read(Base::m_fid); } } @@ -121,7 +121,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cldliq_host(s).write(fid); + cldliq_host(s).write(Base::m_fid); } } } @@ -167,7 +167,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - rain[i].read(fid); + rain[i].read(Base::m_fid); } } @@ -215,7 +215,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - rain_host(s).write(fid); + rain_host(s).write(Base::m_fid); } } } @@ -261,7 +261,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - self[i].read(fid); + self[i].read(Base::m_fid); } } @@ -299,7 +299,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - self_host(s).write(fid); + self_host(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp index 2ef5e66803a..9b261409ad0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp @@ -84,7 +84,7 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - baseline_data[i].read(fid); + baseline_data[i].read(Base::m_fid); } } @@ -151,7 +151,7 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::Un } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cxx_host(s).write(fid); + cxx_host(s).write(Base::m_fid); } } } // run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index 739f2a31629..4e24cc8be0e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -60,7 +60,7 @@ void ice_melting_bfb() { // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - IceMelt[i].read(fid); + IceMelt[i].read(Base::m_fid); } } @@ -109,7 +109,7 @@ void ice_melting_bfb() { } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - IceMelt_host(s).write(fid); + IceMelt_host(s).write(Base::m_fid); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp index f0b4cc4a324..7eeb4fa875e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp @@ -58,7 +58,7 @@ struct UnitWrap::UnitTest::TestIceNucleation : public UnitWrap::UnitTest:: std::string file_name = root_name + (do_predict_nc ? "1" : "0") + (do_prescribed_CCN ? "1" : "0"); if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - self[i].read(fid); + self[i].read(Base::m_fid); } } @@ -105,7 +105,7 @@ struct UnitWrap::UnitTest::TestIceNucleation : public UnitWrap::UnitTest:: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - self_host(s).write(fid); + self_host(s).write(Base::m_fid); } } } //end for do_predict_nc diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp index d0a9b3902f9..59768bbf82c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp @@ -52,7 +52,7 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale : public UnitWrap::Unit // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - self[i].read(fid); + self[i].read(Base::m_fid); } } @@ -103,7 +103,7 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale : public UnitWrap::Unit } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - self_host(s).write(fid); + self_host(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index 6f567a8ae64..45ea1e90323 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -80,7 +80,7 @@ void run_bfb_calc_bulk_rhime() // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - cbrr_baseline[i].read(fid); + cbrr_baseline[i].read(Base::m_fid); } } @@ -122,7 +122,7 @@ void run_bfb_calc_bulk_rhime() } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cbrr_host(s).write(fid); + cbrr_host(s).write(Base::m_fid); } } } @@ -158,7 +158,7 @@ void run_bfb_ice_sed() // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - isds_baseline[i].read(fid); + isds_baseline[i].read(Base::m_fid); } } @@ -193,7 +193,7 @@ void run_bfb_ice_sed() } else if (this->m_baseline_action == GENERATE) { for (Int i = 0; i < num_runs; ++i) { - isds_cxx[i].write(fid); + isds_cxx[i].write(Base::m_fid); } } } @@ -238,7 +238,7 @@ void run_bfb_homogeneous_freezing() // Read baseline data if (this->m_baseline_action == COMPARE) { for (auto& d : hfds_baseline) { - d.read(fid); + d.read(Base::m_fid); } } @@ -269,7 +269,7 @@ void run_bfb_homogeneous_freezing() } else if (this->m_baseline_action == GENERATE) { for (Int i = 0; i < num_runs; ++i) { - hfds_cxx[i].write(fid); + hfds_cxx[i].write(Base::m_fid); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp index 0324bbf1498..b089fdf978f 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp @@ -45,7 +45,7 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::Uni // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - baseline_data[i].read(fid); + baseline_data[i].read(Base::m_fid); } } @@ -88,7 +88,7 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::Uni } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cxx_host(s).write(fid); + cxx_host(s).write(Base::m_fid); } } } // run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp index 22ac35475fb..ea3fdaccf7e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp @@ -212,10 +212,10 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - lid[i].read(fid); - lidb[i].read(fid); - altd[i].read(fid); - altcd[i].read(fid); + lid[i].read(Base::m_fid); + lidb[i].read(Base::m_fid); + altd[i].read(Base::m_fid); + altcd[i].read(Base::m_fid); } } @@ -313,10 +313,10 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base altcd[s].proc = real_results_mirror(6, s); - lid[s].write(fid); - lidb[s].write(fid); - altd[s].write(fid); - altcd[s].write(fid); + lid[s].write(Base::m_fid); + lidb[s].write(Base::m_fid); + altd[s].write(Base::m_fid); + altcd[s].write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp index 79bffe1ac99..eae1454c36a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp @@ -72,7 +72,7 @@ struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest:: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - self[i].read(fid); + self[i].read(Base::m_fid); } } @@ -131,7 +131,7 @@ struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest:: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - self_host(s).write(fid); + self_host(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index 26f075dfef3..e694636806d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -97,7 +97,7 @@ void run_bfb_p3_main_part1() // Read baseline data if (this->m_baseline_action == COMPARE) { for (auto& d : isds_baseline) { - d.read(fid); + d.read(Base::m_fid); } } @@ -151,7 +151,7 @@ void run_bfb_p3_main_part1() } else if (this->m_baseline_action == GENERATE) { for (Int i = 0; i < num_runs; ++i) { - isds_cxx[i].write(fid); + isds_cxx[i].write(Base::m_fid); } } } @@ -206,7 +206,7 @@ void run_bfb_p3_main_part2() // Read baseline data if (this->m_baseline_action == COMPARE) { for (auto& d : isds_baseline) { - d.read(fid); + d.read(Base::m_fid); } } @@ -282,7 +282,7 @@ void run_bfb_p3_main_part2() } else if (this->m_baseline_action == GENERATE) { for (Int i = 0; i < num_runs; ++i) { - isds_cxx[i].write(fid); + isds_cxx[i].write(Base::m_fid); } } } @@ -330,7 +330,7 @@ void run_bfb_p3_main_part3() // Read baseline data if (this->m_baseline_action == COMPARE) { for (auto& d : isds_baseline) { - d.read(fid); + d.read(Base::m_fid); } } @@ -384,7 +384,7 @@ void run_bfb_p3_main_part3() } else if (this->m_baseline_action == GENERATE) { for (Int i = 0; i < num_runs; ++i) { - isds_cxx[i].write(fid); + isds_cxx[i].write(Base::m_fid); } } } @@ -439,7 +439,7 @@ void run_bfb_p3_main() // Read baseline data if (this->m_baseline_action == COMPARE) { for (auto& d : isds_baseline) { - d.read(fid); + d.read(Base::m_fid); } } @@ -498,7 +498,7 @@ void run_bfb_p3_main() } else if (this->m_baseline_action == GENERATE) { for (Int i = 0; i < num_runs; ++i) { - isds_cxx[i].write(fid); + isds_cxx[i].write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp index 288a52c4ad8..04ad1a7f112 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp @@ -38,7 +38,7 @@ struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - baseline_data[i].read(fid); + baseline_data[i].read(Base::m_fid); } } @@ -84,7 +84,7 @@ struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cxx_host(s).write(fid); + cxx_host(s).write(Base::m_fid); } } } // run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp index a721dc7b9af..e165aca01ba 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp @@ -38,7 +38,7 @@ struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - baseline_data[i].read(fid); + baseline_data[i].read(Base::m_fid); } } @@ -83,7 +83,7 @@ struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cxx_host(s).write(fid); + cxx_host(s).write(Base::m_fid); } } } // run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp index e7a75821ae1..982840d2f40 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp @@ -39,7 +39,7 @@ struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - baseline_data[i].read(fid); + baseline_data[i].read(Base::m_fid); } } @@ -88,7 +88,7 @@ struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cxx_host(s).write(fid); + cxx_host(s).write(Base::m_fid); } } } // run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index 803d353eea2..63e40f575e6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -125,7 +125,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - baseline_data[i].read(fid); + baseline_data[i].read(Base::m_fid); } } @@ -178,7 +178,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cxx_host(s).write(fid); + cxx_host(s).write(Base::m_fid); } } } // run_bfb diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp index a391a863c72..935842cb6d9 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp @@ -72,7 +72,7 @@ void run_bfb() // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - rain_imm_freezing_data[i].read(fid); + rain_imm_freezing_data[i].read(Base::m_fid); } } @@ -116,7 +116,7 @@ void run_bfb() } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - host_data(s).write(fid); + host_data(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp index 52ff887168b..26440b6680b 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp @@ -59,7 +59,7 @@ struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - dc[i].read(fid); + dc[i].read(Base::m_fid); } } @@ -103,7 +103,7 @@ struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - dc_host(s).write(fid); + dc_host(s).write(Base::m_fid); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index 5aed6dfe689..91e3db2f297 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -225,7 +225,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - cwdc[i].read(fid); + cwdc[i].read(Base::m_fid); } } @@ -279,7 +279,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - cwdc_host(s).write(fid); + cwdc_host(s).write(Base::m_fid); } } } @@ -322,7 +322,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - iwdc[i].read(fid); + iwdc[i].read(Base::m_fid); } } @@ -381,7 +381,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - iwdc_host(s).write(fid); + iwdc_host(s).write(Base::m_fid); } } } @@ -424,7 +424,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - rwdc[i].read(fid); + rwdc[i].read(Base::m_fid); } } @@ -478,7 +478,7 @@ struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - rwdc_host(s).write(fid); + rwdc_host(s).write(Base::m_fid); } } } @@ -615,7 +615,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitT // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - pupidc[i].read(fid); + pupidc[i].read(Base::m_fid); } } @@ -718,7 +718,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitT } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - pupidc_host(s).write(fid); + pupidc_host(s).write(Base::m_fid); } } } @@ -768,7 +768,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::U // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - gtspvd[i].read(fid); + gtspvd[i].read(Base::m_fid); } } @@ -839,7 +839,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::U } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - gtspvd_host(s).write(fid); + gtspvd_host(s).write(Base::m_fid); } } } @@ -934,7 +934,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitT // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - pupldc[i].read(fid); + pupldc[i].read(Base::m_fid); } } @@ -1020,7 +1020,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitT } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - pupldc_host(s).write(fid); + pupldc_host(s).write(Base::m_fid); } } } @@ -1071,7 +1071,7 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi : public UnitWrap: // Read baseline data if (this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - dc[i].read(fid); + dc[i].read(Base::m_fid); } } @@ -1106,7 +1106,7 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi : public UnitWrap: } else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { - dc_host(s).write(fid); + dc_host(s).write(Base::m_fid); } } } From 037d5af9f188cbc25f40a4338cd6c0acba0321a9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 7 Nov 2024 15:59:40 -0700 Subject: [PATCH 260/366] seds --- .../p3/tests/p3_back_to_cell_average_unit_tests.cpp | 2 +- .../tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp | 2 +- .../src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp | 2 +- .../eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp | 4 ++-- .../p3/tests/p3_ice_supersat_conservation_tests.cpp | 2 +- .../eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp | 8 ++++---- .../src/physics/p3/tests/p3_nc_conservation_tests.cpp | 2 +- .../src/physics/p3/tests/p3_ni_conservation_tests.cpp | 2 +- .../src/physics/p3/tests/p3_nr_conservation_tests.cpp | 2 +- .../p3/tests/p3_prevent_liq_supersaturation_tests.cpp | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index f13a0ca909a..b108ce329d3 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -27,7 +27,7 @@ void run_phys() void run_bfb() { - auto engine = setup_random_test(314552); + auto engine = Base::get_engine(); // Generate n test structs, each populated with random data (values within // [0,1]) by the default constructor. diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index 56c2beafaed..ea217a4d047 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -28,7 +28,7 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: void run_bfb() { - auto engine = setup_random_test(12354); + auto engine = Base::get_engine(); // Read in tables view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index db42d68e6bf..b29bf11faa1 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -27,7 +27,7 @@ void run_phys() void run_bfb() { - auto engine = setup_random_test(23512); + auto engine = Base::get_engine(); CloudSedData csds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, do_predict_nc, precip_liq_surf, diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index 45ea1e90323..faa6da1596e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -129,7 +129,7 @@ void run_bfb_calc_bulk_rhime() void run_bfb_ice_sed() { - auto engine = setup_random_test(124135); + auto engine = Base::get_engine(); IceSedData isds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, precip_ice_surf @@ -202,7 +202,7 @@ void run_bfb_homogeneous_freezing() { constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(13543563); + auto engine = Base::get_engine(); HomogeneousFreezingData hfds_baseline[] = { // kts, kte, ktop, kbot, kdir diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp index b089fdf978f..dcbdfc61826 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp @@ -20,7 +20,7 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::Uni constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(124151); + auto engine = Base::get_engine(); IceSupersatConservationData baseline_data[max_pack_size]; diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index e694636806d..4fb471c3cf5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -50,7 +50,7 @@ void run_phys() void run_bfb_p3_main_part1() { - auto engine = setup_random_test(125125); + auto engine = Base::get_engine(); constexpr Scalar qsmall = C::QSMALL; //PMC wouldn't it make more sense to define qsmall at a higher level since used in part1, part2, and part3? constexpr Scalar T_zerodegc = C::T_zerodegc; @@ -158,7 +158,7 @@ void run_bfb_p3_main_part1() void run_bfb_p3_main_part2() { - auto engine = setup_random_test(263267); + auto engine = Base::get_engine(); constexpr Scalar qsmall = C::QSMALL; constexpr Scalar T_zerodegc = C::T_zerodegc; @@ -292,7 +292,7 @@ void run_bfb_p3_main_part3() constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(3734734); + auto engine = Base::get_engine(); constexpr Scalar qsmall = C::QSMALL; @@ -391,7 +391,7 @@ void run_bfb_p3_main_part3() void run_bfb_p3_main() { - auto engine = setup_random_test(3427727); + auto engine = Base::get_engine(); P3MainData isds_baseline[] = { // its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN diff --git a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp index 04ad1a7f112..175247bb501 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp @@ -17,7 +17,7 @@ struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest: void run_bfb() { - auto engine = setup_random_test(1241556); + auto engine = Base::get_engine(); NcConservationData baseline_data[max_pack_size]; diff --git a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp index e165aca01ba..b8cfdf4333a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp @@ -17,7 +17,7 @@ struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest: void run_bfb() { - auto engine = setup_random_test(1925985); + auto engine = Base::get_engine(); NiConservationData baseline_data[max_pack_size]; diff --git a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp index 982840d2f40..dc9a44af88d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp @@ -17,7 +17,7 @@ struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest: void run_bfb() { - auto engine = setup_random_test(912874); + auto engine = Base::get_engine(); NrConservationData baseline_data[max_pack_size]; diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index 63e40f575e6..1b439aca7aa 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -92,7 +92,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::U constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(1298758); + auto engine = Base::get_engine(); PreventLiqSupersaturationData baseline_data[max_pack_size]; From abf91a6499a917d65c473e1e12579df089dbc2d0 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 7 Nov 2024 16:04:27 -0700 Subject: [PATCH 261/366] Fix bug --- .../p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index ea217a4d047..2752746b792 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -104,7 +104,7 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: REQUIRE(self[s].epsc == self_host(s).epsc); } } - else { + else if (this->m_baseline_action == GENERATE) { for (Int s = 0; s < max_pack_size; ++s) { self_host(s).write(Base::m_fid); } From d01f84440a54cbe1b755b5d193af3814c4e1d586 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 7 Nov 2024 16:23:07 -0700 Subject: [PATCH 262/366] Add ghci-oci --- components/eamxx/scripts/machines_specs.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 41d09f4683e..9827692bfd0 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -225,6 +225,15 @@ def setup(cls): cls.baselines_dir = "/projects/e3sm/baselines/scream/ghci-snl-cuda" cls.gpu_arch = "cuda" +############################################################################### +class GHCIOCI(Machine): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_base(name="ghci-oci") + cls.env_setup = [f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-oci_gnu)"] + ############################################################################### class Lassen(Machine): ############################################################################### From 23dbd2eddad751a112a049afaee8951ff09a2295 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 7 Nov 2024 16:50:51 -0700 Subject: [PATCH 263/366] Minor fixes --- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 14 ++++++++------ .../eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp | 13 ++++++++++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 70ac3947fa8..784d3436c88 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -83,12 +83,14 @@ if (NOT SCREAM_P3_SMALL_KERNELS) # Make sure that a diff in the two implementation triggers a failed test (in debug only) # No need to run lots of different thread counts. - CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3_sk p3_test_infra - EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - LABELS "p3_sk;physics;fail" - ${FORCE_RUN_DIFF_FAILS}) + if (SCREAM_ENABLE_BASELINE_TESTS) + CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp + LIBS p3_sk p3_test_infra + EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + LABELS "p3_sk;physics;fail" + ${FORCE_RUN_DIFF_FAILS}) + endif() endif() CreateUnitTest(p3_run_and_cmp "p3_run_and_cmp.cpp" diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index 4c0d5a214e0..f6d9be48c2a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -122,8 +122,11 @@ struct Baseline { } Int run_and_cmp (const std::string& filename, const double& tol, bool no_baseline) { - auto fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); - EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + ekat::FILEPtr fid; + if (no_baseline) { + fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); + EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + } Int nerr = 0, ne; int case_num = 0; for (auto ps : params_) { @@ -352,7 +355,11 @@ int main (int argc, char** argv) { if (generate) { std::cout << "Generating to " << baseline_fn << "\n"; nerr += bln.generate_baseline(baseline_fn); - } else { + } else if (no_baseline) { + printf("Running with no baseline actions\n"); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); + } + else { printf("Comparing with %s at tol %1.1e\n", baseline_fn.c_str(), tol); nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); } From 03630d39835cdc50f9347982acc2603abb282d64 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 7 Nov 2024 17:16:37 -0700 Subject: [PATCH 264/366] Another minor fix --- components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index f6d9be48c2a..ed6f343fbcf 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -123,7 +123,7 @@ struct Baseline { Int run_and_cmp (const std::string& filename, const double& tol, bool no_baseline) { ekat::FILEPtr fid; - if (no_baseline) { + if (!no_baseline) { fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); } From a17807865bf2ac7c5a19eb5e07724bb5d13cbb74 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 8 Nov 2024 13:41:22 -0700 Subject: [PATCH 265/366] EAMxx: fix query-scream util script It's used by ekat when testing a PR --- components/eamxx/scripts/query_scream.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/components/eamxx/scripts/query_scream.py b/components/eamxx/scripts/query_scream.py index 4b26451c7cb..a6fe5096bfb 100644 --- a/components/eamxx/scripts/query_scream.py +++ b/components/eamxx/scripts/query_scream.py @@ -1,9 +1,5 @@ -from machines_specs import assert_machine_supported, \ - get_mach_cxx_compiler, get_mach_c_compiler, get_mach_f90_compiler, \ - get_mach_batch_command, get_mach_env_setup_command, \ - get_mach_baseline_root_dir, is_cuda_machine, \ - get_mach_compilation_resources, get_mach_testing_resources +from machines_specs import assert_machine_supported, get_machine, get_mach_env_setup_command from utils import expect CHOICES = ( @@ -24,23 +20,24 @@ def query_scream(machine, param): assert_machine_supported(machine) expect(param in CHOICES, f"Unknown param {param}") + mach = get_machine(machine) if param == "cxx_compiler": - return get_mach_cxx_compiler(machine) + return mach.cxx_compiler elif param == "c_compiler": - return get_mach_c_compiler(machine) + return mach.c_compiler elif param == "f90_compiler": - return get_mach_f90_compiler(machine) + return mach.ftn_compiler elif param == "batch": - return get_mach_batch_command(machine) + return mach.batch elif param == "env": return get_mach_env_setup_command(machine) elif param == "baseline_root": - return get_mach_baseline_root_dir(machine) + return mach.baselines_dir elif param == "cuda": - return str(is_cuda_machine(machine)) + return str(mach.gpu_arch == "cuda") elif param == "comp_j": - return get_mach_compilation_resources() + return num_bld_res elif param == "test_j": - return get_mach_testing_resources(machine) + return gnum_run_res else: expect(False, f"Unhandled param {param}") From 579850c0776cfbd80f8c55fe1b34728a3ab54c01 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 8 Nov 2024 15:38:42 -0700 Subject: [PATCH 266/366] Update .mergify.yml Fix required checks names --- .mergify.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index e931d54eb09..6c9318ce5b6 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,21 +1,20 @@ merge_protections: - name: Enforce checks passing description: Make sure that checks are not failing on the PR, and reviewers approved - comment: false if: - base = master success_conditions: - "#approved-reviews-by >= 1" # At least 1 approval - "#changes-requested-reviews-by == 0" # No reviewer asked for changes - or: - - check-success=eamxx-sa/gcc-openmp/* - - check-skipped=eamxx-sa/gcc-openmp/* + - check-success="gcc-openmp / .*" + - check-skipped="gcc-openmp / .*" - or: - - check-success=eamxx-sa/gcc-cuda/* - - check-skipped=eamxx-sa/gcc-cuda/* + - check-success="gcc-cuda / .*" + - check-skipped="gcc-cuda / .*" - or: - - check-success=eamxx-v1/cpu-gcc/* - - check-skipped=eamxx-v1/cpu-gcc/* + - check-success="cpu-gcc / .*" + - check-skipped="cpu-gcc / .*" - or: - - check-success=eamxx-scripts/cpu-gcc - - check-skipped=eamxx-scripts/cpu-gcc + - check-success=cpu-gcc + - check-skipped=cpu-gcc From fefea965df034141d5bed9999db9295aa897ef68 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 8 Nov 2024 15:17:22 -0700 Subject: [PATCH 267/366] Update ekat submodule --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index 4e36a487d9c..1d441b22df3 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 4e36a487d9ced3951907b36a1a16b13325d796d1 +Subproject commit 1d441b22df3e4f8f8b3ea96099b0e848eb74afd7 From acd26a119487f9c3fe66ae4cb4063f709cce98e9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 8 Nov 2024 15:18:38 -0700 Subject: [PATCH 268/366] EAMxx: adapt to changes in ekat's catch main logic --- .../src/physics/p3/tests/p3_run_and_cmp.cpp | 38 ++++--------------- .../src/physics/rrtmgp/tests/rrtmgp_tests.cpp | 14 +++---- .../physics/shoc/tests/shoc_run_and_cmp.cpp | 38 ++++--------------- .../eamxx/src/share/util/eamxx_ad_test.cpp | 2 +- .../model_restart/CMakeLists.txt | 6 +-- .../atm_proc_subcycling/CMakeLists.txt | 4 +- .../shoc_p3_nudging/CMakeLists.txt | 8 ++-- .../single-process/rrtmgp/CMakeLists.txt | 6 +-- .../rrtmgp/rrtmgp_standalone_unit.cpp | 4 +- 9 files changed, 34 insertions(+), 86 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index 5bc12b2464d..ed252b5034c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -279,10 +279,9 @@ int main (int argc, char** argv) { ++i; nlev = std::atoi(argv[i]); } - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + auto tokens = ekat::split(argv[i],"="); + device = tokens[1]; } if (ekat::argv_matches(argv[i], "-r", "--repeat")) { expect_another_arg(i, argc); @@ -311,32 +310,8 @@ int main (int argc, char** argv) { // Decorate baseline name with precision. baseline_fn += std::to_string(sizeof(scream::Real)); - std::vector args; - for (int i=0; i" was specified, add kokkos - // initialization flag to argv - // Create it outside the if, so its c_str pointer survives - std::string dev_arg; - if (device!="") { - auto is_int = [] (const std::string& s)->bool { - std::istringstream is(s); - int d; - is >> d; - return !is.fail() && is.eof(); - }; - - EKAT_REQUIRE_MSG (is_int(device), "Error! Invalid device specification.\n"); - - if (std::stoi(device) != -1) { - dev_arg = "--kokkos-device-id=" + device; - args.push_back(const_cast(dev_arg.c_str())); - } - } - - scream::initialize_scream_session(args.size(), args.data()); { + scream::initialize_scream_session(argc, argv); + { Baseline bln(timesteps, static_cast(dt), ncol, nlev, repeat, predict_nc, prescribed_ccn); if (generate) { std::cout << "Generating to " << baseline_fn << "\n"; @@ -346,7 +321,8 @@ int main (int argc, char** argv) { nerr += bln.run_and_cmp(baseline_fn, tol, use_fortran); } P3GlobalForFortran::deinit(); - } scream::finalize_scream_session(); + } + scream::finalize_scream_session(); return nerr != 0 ? 1 : 0; } diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 0d3f18e7d84..fedb8c3d047 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -46,7 +46,7 @@ int run_yakl(int argc, char** argv) { logger->error(msg); return 1; } - std::string inputfile, baseline, device; + std::string inputfile, baseline; for (int i = 1; i < argc-1; ++i) { if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { @@ -60,10 +60,8 @@ int run_yakl(int argc, char** argv) { inputfile = argv[i]; } // RRTMGP baselines tests to not use kokoks. Swallow the arg, but ignore it - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + continue; } } @@ -352,10 +350,8 @@ int run_kokkos(int argc, char** argv) { inputfile = argv[i]; } // RRTMGP baselines tests to not use kokoks. Swallow the arg, but ignore it - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + continue; } } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp b/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp index a5198d95980..94a58b4de9c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp @@ -271,42 +271,17 @@ int main (int argc, char** argv) { generate = true; } } - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + auto tokens = ekat::split(argv[i],"="); + device = tokens[1]; } } // Decorate baseline name with precision. baseline_fn += std::to_string(sizeof(scream::Real)); - std::vector args; - for (int i=0; i" was specified, add kokkos - // initialization flag to argv - // Create it outside the if, so its c_str pointer survives - std::string dev_arg; - if (device!="") { - auto is_int = [] (const std::string& s)->bool { - std::istringstream is(s); - int d; - is >> d; - return !is.fail() && is.eof(); - }; - - EKAT_REQUIRE_MSG (is_int(device), "Error! Invalid device specification.\n"); - - if (std::stoi(device) != -1) { - dev_arg = "--kokkos-device-id=" + device; - args.push_back(const_cast(dev_arg.c_str())); - } - } - - scream::initialize_scream_session(args.size(), args.data()); { + scream::initialize_scream_session(argc, argv); + { Baseline bln(nsteps, static_cast(dt), ncol, nlev, num_qtracers, nadv, repeat); if (generate) { std::cout << "Generating to " << baseline_fn << "\n"; @@ -315,7 +290,8 @@ int main (int argc, char** argv) { printf("Comparing with %s at tol %1.1e\n", baseline_fn.c_str(), tol); nerr += bln.run_and_cmp(baseline_fn, tol, use_fortran); } - } scream::finalize_scream_session(); + } + scream::finalize_scream_session(); return nerr != 0 ? 1 : 0; } diff --git a/components/eamxx/src/share/util/eamxx_ad_test.cpp b/components/eamxx/src/share/util/eamxx_ad_test.cpp index 0f64bd7c5ab..ba493478878 100644 --- a/components/eamxx/src/share/util/eamxx_ad_test.cpp +++ b/components/eamxx/src/share/util/eamxx_ad_test.cpp @@ -32,7 +32,7 @@ TEST_CASE("scream_ad_test") { // Create a comm ekat::Comm atm_comm (MPI_COMM_WORLD); - // User can prescribe input file name via --ekat-test-params ifile= + // User can prescribe input file name via --args --ifile auto& session = ekat::TestSession::get(); session.params.emplace("ifile","input.yaml"); std::string fname = session.params["ifile"]; diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt index 5645a9383a7..a0288b454b2 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt @@ -24,20 +24,20 @@ set (CASE_TN 2023-01-01-00060) # Create the baseline (run all 6 timsteps in a single run) CreateUnitTestFromExec(model_baseline model_restart - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_baseline.yaml" + EXE_ARGS "--args -ifile=input_baseline.yaml" MPI_RANKS ${SCREAM_TEST_MAX_RANKS} FIXTURES_SETUP baseline_run) # Start a simulation, but only run half of the time steps CreateUnitTestFromExec(model_initial model_restart - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_initial.yaml" + EXE_ARGS "--args -ifile=input_initial.yaml" MPI_RANKS ${SCREAM_TEST_MAX_RANKS} FIXTURES_SETUP initial_run PROPERTIES RESOURCE_LOCK rpointer_file) # Restart the simulation, and run the second half of the time steps CreateUnitTestFromExec(model_restart model_restart - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_restarted.yaml" + EXE_ARGS "--args -ifile=input_restarted.yaml" MPI_RANKS ${SCREAM_TEST_MAX_RANKS} FIXTURES_REQUIRED initial_run FIXTURES_SETUP restarted_run diff --git a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt index 71fe2a5f43c..88c5f1022f6 100644 --- a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt @@ -23,7 +23,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output_tend.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_tend_subcycled.yaml) CreateUnitTestFromExec (shoc_p3_subcycled shoc_p3 - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_subcycled.yaml" + EXE_ARGS "--args -ifile=input_subcycled.yaml" FIXTURES_SETUP shoc_p3_subcycled) # Run a test without subcycling and more steps @@ -38,7 +38,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output_tend.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_tend_monolithic.yaml) CreateUnitTestFromExec (shoc_p3_monolithic shoc_p3 - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_monolithic.yaml" + EXE_ARGS "--args -ifile=input_monolithic.yaml" FIXTURES_SETUP shoc_p3_monolithic) # Finally, compare output of the two tests diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt index 62c4b6d2266..16568dc55b3 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt @@ -27,7 +27,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output_remapped.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_source_data_remapped.yaml) CreateUnitTestFromExec (shoc_p3_source shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_source_data.yaml" + EXE_ARGS "--args -ifile=input_source_data.yaml" FIXTURES_SETUP shoc_p3_source_data FIXTURES_REQUIRED shoc_p3_create_vertical_remap_and_weights_file) @@ -41,7 +41,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_nudged.yaml) CreateUnitTestFromExec (shoc_p3_nudged shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging.yaml" + EXE_ARGS "--args -ifile=input_nudging.yaml" FIXTURES_REQUIRED shoc_p3_source_data) # Run a test with nudging turned on using remapped source data for nudging: @@ -54,7 +54,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_nudged_remapped.yaml) CreateUnitTestFromExec (shoc_p3_nudged_remapped shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_remapped.yaml" + EXE_ARGS "--args -ifile=input_nudging_remapped.yaml" FIXTURES_REQUIRED shoc_p3_source_data) # Run a test with nudging using data read in glob pattern and skip vertical interpolation: @@ -67,6 +67,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging_glob_novert.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_nudged_glob_novert.yaml) CreateUnitTestFromExec (shoc_p3_nudging_glob_novert shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_glob_novert.yaml" + EXE_ARGS "--args -ifile=input_nudging_glob_novert.yaml" FIXTURES_REQUIRED shoc_p3_source_data) diff --git a/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt index 792d3946a4c..b16c2e73288 100644 --- a/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt @@ -13,7 +13,7 @@ if (SCREAM_ENABLE_BASELINE_TESTS AND NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(${TEST_BASE_NAME}_unit rrtmgp_standalone_unit.cpp LABELS rrtmgp physics driver LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics rrtmgp_test_utils - EXE_ARGS "--ekat-test-params rrtmgp_inputfile=${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc,rrtmgp_baseline=${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" + EXE_ARGS "--args --inputfile ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc --baseline ${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" ) endif() @@ -39,7 +39,7 @@ CreateUnitTestFromExec( ${TEST_BASE_NAME}_not_chunked ${TEST_BASE_NAME} LABELS rrtmgp physics driver MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_not_chunked.yaml" + EXE_ARGS "--args -inputfile input_not_chunked.yaml" FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_not_chunked ) @@ -70,7 +70,7 @@ CreateUnitTestFromExec( ${TEST_BASE_NAME}_chunked ${TEST_BASE_NAME} LABELS rrtmgp physics driver MPI_RANKS ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_chunked.yaml" + EXE_ARGS "--args --inputfile=input_chunked.yaml" FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_chunked PROPERTIES PASS_REGULAR_EXPRESSION "(beg.end: 0, ${COL_CHUNK_SIZE})" ) diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index dc1ece59b28..5be27ce2b1e 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -49,8 +49,8 @@ using PC = scream::physics::Constants; #ifdef RRTMGP_ENABLE_YAKL TEST_CASE("rrtmgp_scream_standalone", "") { // Get baseline name (needs to be passed as an arg) - std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); - std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); + std::string inputfile = ekat::TestSession::get().params.at("inputfile"); + std::string baseline = ekat::TestSession::get().params.at("baseline"); // Check if files exists REQUIRE(rrtmgpTest::file_exists(inputfile.c_str())); From 067fa9301bd2598fc66e2db91391c2e883ef565b Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 9 Nov 2024 13:13:53 -0600 Subject: [PATCH 269/366] update pam to match new p3 signature --- components/eam/src/physics/crm/pam/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index c3b6522c57b..1c37054d1ff 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit c3b6522c57b754d073e1deaad5ce8125f7f88325 +Subproject commit 1c37054d1ff9b160290cc286dcbd3cdc6cd7e7f6 From 4cf27bcb7fc4cca58e02c34a6d348797537035a5 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 9 Nov 2024 13:35:11 -0600 Subject: [PATCH 270/366] fix formatting of changed md files --- components/eamxx/docs/developer/ci_nightly.md | 6 +- .../eamxx/docs/developer/cime_testing.md | 33 +-- components/eamxx/docs/developer/field.md | 95 +++++---- components/eamxx/docs/developer/grid.md | 43 ++-- components/eamxx/docs/developer/index.md | 2 - components/eamxx/docs/developer/io.md | 6 +- .../eamxx/docs/developer/kokkos_ekat.md | 196 ++++++++++++------ components/eamxx/docs/developer/managers.md | 2 +- components/eamxx/docs/developer/processes.md | 147 +++++++------ .../eamxx/docs/developer/source_tree.md | 1 - .../docs/developer/standalone_testing.md | 37 ++-- .../eamxx/docs/developer/style_guide.md | 1 - 12 files changed, 345 insertions(+), 224 deletions(-) diff --git a/components/eamxx/docs/developer/ci_nightly.md b/components/eamxx/docs/developer/ci_nightly.md index b222139dd55..0716ce4c9f3 100644 --- a/components/eamxx/docs/developer/ci_nightly.md +++ b/components/eamxx/docs/developer/ci_nightly.md @@ -1,17 +1,17 @@ # Continuous Integration and Nightly Testing -## Autotester ## +## Autotester EAMxx using github actions and a Sandia product called Autotester 2 to run CI testing on a CPU and GPU machine for every github pull request. By default, we run the e3sm_scream_v1_at suite and the standalone eamxx tests (test-all-scream). -## Nightly overview, CDash ## +## Nightly overview, CDash Our nightly testing is much more extensive than the CI testing. You can see our dashboard here under the section "E3SM_SCREAM": -https://my.cdash.org/index.php?project=E3SM + We run a variety of CIME test suites and standalone testing on a number of platforms. We even do some performance testing on frontier. diff --git a/components/eamxx/docs/developer/cime_testing.md b/components/eamxx/docs/developer/cime_testing.md index 71233a245b4..667488960f6 100644 --- a/components/eamxx/docs/developer/cime_testing.md +++ b/components/eamxx/docs/developer/cime_testing.md @@ -4,33 +4,37 @@ Full model system testing of eamxx is done through CIME test cases (much like the rest of E3SM). We offer a number of test suites, including: + * e3sm_scream_v0: Test the full set of V0 (pre-C++) tests * e3sm_scream_v1: Test the full set of V1 (C++) tests * e3sm_scream_v1_at: A smaller and quicker set of tests for autotesting * e3sm_scream_hires: A small number of bigger, longer-running tests to measure performance Example for running a suite: -``` -% cd $repo/cime/scripts -% ./create_test e3sm_scream_v1_at --wait + +```shell +cd $repo/cime/scripts +./create_test e3sm_scream_v1_at --wait ``` Example for running a single test case: -``` -% cd $repo/cime/scripts -% ./create_test SMS.ne4_ne4.F2010-SCREAMv1 --wait + +```shell +cd $repo/cime/scripts +./create_test SMS.ne4_ne4.F2010-SCREAMv1 --wait ``` There are many behavioral tweaks you can make to a test case, like changing the run length, test type, etc. Most of this is not specific to eamxx and works for any CIME case. This generic stuff is well-documentated here: -http://esmci.github.io/cime/versions/master/html/users_guide/testing.html + When it comes to things specific to eamxx, you have grids, compsets, and testmods. Common EAMxx grids are: + * ne4_ne4 (low resolution) * ne4pg2_ne4pg2 (low resolution with phys grid) * ne30_ne30 (med resolution) @@ -38,9 +42,10 @@ Common EAMxx grids are: * ne1024pg2_ne1024pg2 (ultra high with phys grid) More grid info can be found here: -https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/933986549/ATM+Grid+Resolution+Summary + Common EAMxx compsets are: + * F2010-SCREAM-LR: V0 low res compset with eamxx V0 atmosphere * F2010-SCREAMv1: V1 standard compset with eamxx V1 atmosphere * FIOP-SCREAMv1-DP: V1 with dpxx (doubly-periodic lateral boundary condition in C++) @@ -50,10 +55,14 @@ Full info on supported compsets can be found by looking at this file: `$scream_repo/components/eamxx/cime_config/config_compsets.xml` Common EAMxx testmods are: -* small_kernels: Enable smaller-granularity kernels, can improve performance on some systems -* scream-output-preset-[1-6]: Our 6 output presets. These turn some combination of our three output streams (phys_dyn, phys, and diags), + +* small_kernels: Enable smaller-granularity kernels, + can improve performance on some systems +* scream-output-preset-[1-6]: Our 6 output presets. + These turn some combination of our three output streams + (phys_dyn, phys, and diags), various remaps, etc. -* bfbhash: Turns on bit-for-bit hash output: https://acme-climate.atlassian.net/wiki/spaces/NGDNA/pages/3831923056/EAMxx+BFB+hashing +* bfbhash: Turns on bit-for-bit hash output: More info on running EAMxx can be found here: -https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3386015745/How+To+Run+EAMxx+SCREAMv1 + diff --git a/components/eamxx/docs/developer/field.md b/components/eamxx/docs/developer/field.md index 4170b28ac2b..8df83440a2f 100644 --- a/components/eamxx/docs/developer/field.md +++ b/components/eamxx/docs/developer/field.md @@ -1,45 +1,58 @@ -## Field +# Field -In EAMxx, a `Field` is a data structure holding two things: pointers to the data and pointers to metadata. -Both the data and metadata are stored in `std::shared_ptr` instances, to ensure consistency across all copies -of the field. This allows for fast shallow copy semantic for this class. +In EAMxx, a `Field` is a data structure holding two things: pointers to the +data and pointers to metadata. Both the data and metadata are stored in +`std::shared_ptr` instances, to ensure consistency across all copies of +the field. This allows for fast shallow copy semantic for this class. -The data is stored on both CPU and device memory (these may be the same, depending on the Kokkos -backend). In EAMxx, we always assume and guarantee that the device data is up to date. That implies that the data -must be explicitly synced to host before using it on host, and explicitly synced to device after host manipulation, -in order to ensure correctness. In order to access the data, users must use the `get_view`/'get_strided_view' methods, -which takes two template arguments: the data type, and an enum specifying whether CPU or device data is needed. -The data type is used to reinterpret the generic pointer stored inside to a view of the correct scalar type and layout. -It is a possibly const-qualified type, and if the field was marked as "read-only", the method ensures that the -provided data type is const. A read-only field can be created via the `getConst` method, which returns a shallow -copy of the field, but marked as read-only. The enum specifying host or device data is optional, with device being the default. +The data is stored on both CPU and device memory (these may be the same, +depending on the Kokkos backend). In EAMxx, we always assume and guarantee +that the device data is up to date. That implies that the data must be +explicitly synced to host before using it on host, and explicitly synced +to device after host manipulation, in order to ensure correctness. +In order to access the data, users must use the `get_view`/ +`get_strided_view` methods, which takes two template arguments: +the data type, and an enum specifying whether CPU or device data is needed. +The data type is used to reinterpret the generic pointer stored inside +to a view of the correct scalar type and layout. It is a possibly +const-qualified type, and if the field was marked as "read-only", +the method ensures that the provided data type is const. A read-only field +can be created via the `getConst` method, which returns a shallow copy of +the field, but marked as read-only. The enum specifying host or device data +is optional, with device being the default. -The metadata is a collection of information on the field, such as name, layout, units, allocation size, and more. -Part of the metadata is immutable after creation (e.g., name, units, or layout), while some metadata can be -partially or completely modified. The metadata is contained in the `FieldHeader` data structure, which contains -four parts: +The metadata is a collection of information on the field, such as name, layout, units, +allocation size, and more. Part of the metadata is immutable after creation (e.g., +name, units, or layout), while some metadata can be partially or completely modified. +The metadata is contained in the `FieldHeader` data structure, which contains four +parts: -* `FieldIdentifier`: stores the field's name, layout, units, data type, and name of the grid where it's defined. - These information are condensed in a single string, that can be used to uniquely identify a field, - allowing to distinguish between different version of the same field. The layout is stored in the `FieldLayout` - data structure, which includes: - * the field tags: stored as a `std::vector`, they give context to the field's extents. - * the field dims: stored both as a `std::vector`, as well as a 1d `Kokkos::View`. -* `FieldTracking`: stores information on the usage of the field, as well as its possible connections to other - fields. In particular, the tracked items are: - * the field time stamp: the time stamp when the field was last updated. - * the field accumulation start time: used for fields that are accumulated over several time steps - (or time step subcycles). For instance, it allows to reconstruct fluxes from raw accumulations. - * the providers/customers: lists of atmosphere processes (see below) that respectively require/compute - the field in their calculations. - * the field groups: a list of field groups that this field belongs too. Field groups are used to access - a group of fields without explicit prior knowledge about the number and/or names of the fields. -* `FieldAllocProp`: stores information about the allocation. While the field is not yet allocated, users can - request special allocations for the field, for instance to accommodate packing (for SIMD), which may - require padding. Upon allocation, this information is then used by the Field structure to extract the - actual data, wrapped in a properly shaped `Kokkos::View`. The alloc props are also responsible of tracking - additional information in case the field is a "slice" of a higher-dimensional one, a fact that can affect - how the data is accessed. -* Extra data: stored as a `std::map`, allows to catch any metadata that does not fit - in the above structures. This is a last resort structure, intended to accommodate the most peculiar - corner cases, and should be used sparingly. +* `FieldIdentifier`: stores the field's name, layout, units, data type, + and name of the grid where it's defined. These information are condensed + in a single string, that can be used to uniquely identify a field, allowing + to distinguish between different version of the same field. + The layout is stored in the `FieldLayout` data structure, which includes: + * the field tags: stored as a `std::vector`, they give context to the + field's extents. + * the field dims: stored both as a `std::vector`, as well as a 1d `Kokkos::View`. +* `FieldTracking`: stores information on the usage of the field, as well as its + possible connections to other fields. In particular, the tracked items are: + * the field time stamp: the time stamp when the field was last updated. + * the field accumulation start time: used for fields that are accumulated over + several time steps (or time step subcycles). For instance, it allows to + reconstruct fluxes from raw accumulations. + * the providers/customers: lists of atmosphere processes (see below) that + respectively require/compute the field in their calculations. + * the field groups: a list of field groups that this field belongs too. Field groups + are used to access a group of fields without explicit prior knowledge about the + number and/or names of the fields. +* `FieldAllocProp`: stores information about the allocation. While the field is not + yet allocated, users can request special allocations for the field, for instance + to accommodate packing (for SIMD), which may require padding. Upon allocation, + this information is then used by the Field structure to extract the actual data, + wrapped in a properly shaped `Kokkos::View`. The alloc props are also + responsible of tracking additional information in case the field is a "slice" of + a higher-dimensional one, a fact that can affect how the data is accessed. +* Extra data: stored as a `std::map`, allows to catch any + metadata that does not fit in the above structures. This is a last resort structure, + intended to accommodate the most peculiar corner cases, and should be used sparingly. diff --git a/components/eamxx/docs/developer/grid.md b/components/eamxx/docs/developer/grid.md index 8a61b97e079..b4e1a1c8c03 100644 --- a/components/eamxx/docs/developer/grid.md +++ b/components/eamxx/docs/developer/grid.md @@ -1,22 +1,29 @@ -## Grids and Remappers +# Grids and Remappers -In EAMxx, the `AbstractGrid` is an interface used to access information regarding the horizontal and vertical -discretization. The most important information that the grid stores is: +In EAMxx, the `AbstractGrid` is an interface used to access information regarding +the horizontal and vertical discretization. The most important information that +the grid stores is: -* the number of local/global DOFs: these are the degrees of freedom of the horizontal grid only. Here, - local/global refers to the MPI partitioning. -* the DOFs global IDs (GIDs): a list of GIDs of the DOFs on the current MPI rank, stored as a Field -* the local IDs (LIDs) to index list: this list maps the LID of a DOF (that is, the position of the DOF - in the GID list) to a "native" indexing system for that DOF. For instance, a `PointGrid` (a class derived from - `AbstractGrid`) is a simple collection of points, so the "native" indexing system coincides with the LIDs. - However, for a `SEGrid` (a derived class, for spectral element grids), the "native" indexing is a triplet - `(ielem,igp,jgp)`, specifying the element index, and the two indices of the Gauss point within the element. -* geometry data: stored as a `std::map`, this represent any data that is intrinsically - linked to the grid (either along the horizontal or vertical direction), such as lat/lon coordinates, - vertical coordinates, area associated with the DOF. +* the number of local/global DOFs: these are the degrees of freedom of the + horizontal grid only. Here, local/global refers to the MPI partitioning. +* the DOFs global IDs (GIDs): a list of GIDs of the DOFs on the current MPI rank, + stored as a Field +* the local IDs (LIDs) to index list: this list maps the LID of a DOF (that is, + the position of the DOF in the GID list) to a "native" indexing system for that + DOF. For instance, a `PointGrid` (a class derived from `AbstractGrid`) is a + simple collection of points, so the "native" indexing system coincides with the + LIDs. However, for a `SEGrid` (a derived class, for spectral element grids), + the "native" indexing is a triplet `(ielem,igp,jgp)`, specifying the element + index, and the two indices of the Gauss point within the element. +* geometry data: stored as a `std::map`, this represent any + data that is intrinsically linked to the grid (either along the horizontal or + vertical direction), such as lat/lon coordinates, vertical coordinates, area + associated with the DOF. -Grids can also be used to retrieve the layout of a 2d/3d scalar/vector field, which allows certain downstream -classes to perform certain operations without assuming anything on the horizontal grid. +Grids can also be used to retrieve the layout of a 2d/3d scalar/vector field, +which allows certain downstream classes to perform certain operations without +assuming anything on the horizontal grid. -In general, grid objects are passed around the different parts of EAMxx as const objects (read-only). -The internal data can only be modified during construction, which usually is handled by a `GridsManager` object. +In general, grid objects are passed around the different parts of EAMxx as const +objects (read-only). The internal data can only be modified during construction, +which usually is handled by a `GridsManager` object. diff --git a/components/eamxx/docs/developer/index.md b/components/eamxx/docs/developer/index.md index 2d47bab65fe..69673b12ebd 100644 --- a/components/eamxx/docs/developer/index.md +++ b/components/eamxx/docs/developer/index.md @@ -1,3 +1 @@ # SCREAM Developer Guide - - diff --git a/components/eamxx/docs/developer/io.md b/components/eamxx/docs/developer/io.md index caf237010a3..0a4c7b2d832 100644 --- a/components/eamxx/docs/developer/io.md +++ b/components/eamxx/docs/developer/io.md @@ -1,5 +1,5 @@ # Input-Output -In EAMxx, I/O is handled through the SCORPIO library, currently a submodule of E3SM. -The `scream_io` library within eamxx allows to interface the EAMxx infrastructure classes -with the SCORPIO library. +In EAMxx, I/O is handled through the SCORPIO library, currently a submodule of +E3SM. The `scream_io` library within eamxx allows to interface the EAMxx +infrastructure classes with the SCORPIO library. diff --git a/components/eamxx/docs/developer/kokkos_ekat.md b/components/eamxx/docs/developer/kokkos_ekat.md index 45827a11f83..2432290a67a 100644 --- a/components/eamxx/docs/developer/kokkos_ekat.md +++ b/components/eamxx/docs/developer/kokkos_ekat.md @@ -2,99 +2,163 @@ ## Kokkos -EAMxx uses Kokkos for performance portable abstractions for parallel execution of code and data management to various HPC platforms, including OpenMP, Cuda, HIP, and SYCL. Here we give a brief overview of the important concepts for understanding Kokkos in EAMxx. For a more in depth description, see the [Kokkos wiki](https://kokkos.org/kokkos-core-wiki). +EAMxx uses Kokkos for performance portable abstractions for parallel execution +of code and data management to various HPC platforms, including OpenMP, Cuda, +HIP, and SYCL. Here we give a brief overview of the important concepts for +understanding Kokkos in EAMxx. For a more in depth description, see the +[Kokkos wiki](https://kokkos.org/kokkos-core-wiki). ### Kokkos::Device -`Kokkos::Device` is a struct which contain the type definitions for two main Kokkos concepts: execution space (`Kokkos::Device::execution_space`), the place on-node where parallel operations (like for-loops, reductions, etc.) are executed, and the memory space (`Kokkos::Device::memory_space`), the memory location on-node where data is stored. Given your machine architecture, Kokkos defines a default "device" space, given by -``` -Kokkos::Device -``` -where all performance critical code should be executed (e.g., on an NVIDIA machine, this device would be the GPU accelerators) and a default "host" space, given by +`Kokkos::Device` is a struct which contain the type definitions for two main +Kokkos concepts: execution space (`Kokkos::Device::execution_space`), the place +on-node where parallel operations (like for-loops, reductions, etc.) are +executed, and the memory space (`Kokkos::Device::memory_space`), the memory +location on-node where data is stored. Given your machine architecture, Kokkos +defines a default "device" space, given by + +```cpp +Kokkos::Device ``` -Kokkos::Device + +where all performance critical code should be executed (e.g., on an NVIDIA +machine, this device would be the GPU accelerators) and a default "host" space, +given by + +```c++ +Kokkos::Device ``` -where data can be accessed by the CPU cores and is necessary for I/O interfacing, for example. Currently, these default spaces are the ones used by EAMxx. On CPU-only machines, host and device represent the same space. + +where data can be accessed by the CPU cores and is necessary for I/O +interfacing, for example. Currently, these default spaces are the ones used by +EAMxx. On CPU-only machines, host and device represent the same space. ### Kokkos Views -The main data struct provided by Kokkos used in EAMxx in the `Kokkos::View`. This is a multi-dimensional data array that can live on either device or host memory space. These Views are necessary when running on GPU architectures as data structures like `std::vector` and `std::array` will be unavailable on device. +The main data struct provided by Kokkos used in EAMxx in the `Kokkos::View`. +This is a multi-dimensional data array that can live on either device or host +memory space. These Views are necessary when running on GPU architectures as +data structures like `std::vector` and `std::array` will be unavailable on +device. -Views are constructed in EAMxx most commonly with the following template and input arguments -``` -Kokkos::View(const std::string& label, int dim0, int dim1, ...) +Views are constructed in EAMxx most commonly with the following template and +input arguments + +```cpp +Kokkos::View(const std::string& label, + int dim0, int dim1, ...) ``` + where - - `DataType`: scalar type of the view, given as `ScalarType`+`*`(x's number of run-time dimensions). E.g., a 2D view of doubles will have `DataType = double**`. There is also an ability to define compile-time dimensions by using `[]`, see [Kokkos wiki section on views](https://kokkos.org/kokkos-core-wiki/API/core/view/view.html). - - `LayoutType`: mapping of indices into the underlying 1D memory storage. Types are: - - `LayoutRight` (used in EAMxx): strides increase from the right most to the left most dimension, right-most dimension is contiguous - - `LayoutLeft`: strides increase from the left most to the right most dimension, left-most dimension is contiguous - - `LayoutStride`: strides can be arbitrary for each dimension - - `DeviceType`: provides space where data live, defaults to the default device +- `DataType`: scalar type of the view, given as `ScalarType`+`*`(x's number of + run-time dimensions). E.g., a 2D view of doubles will have `DataType = + double**`. There is also an ability to define compile-time dimensions by + using `[]`, see [Kokkos wiki section on views]( + wiki/API/core/view/view.html). +- `LayoutType`: mapping of indices into the underlying 1D memory storage. Types + are: + - `LayoutRight` (used in EAMxx): strides increase from the right most to the + left most dimension, right-most dimension is contiguous + - `LayoutLeft`: strides increase from the left most to the right most + dimension, left-most dimension is contiguous + - `LayoutStride`: strides can be arbitrary for each dimension +- `DeviceType`: provides space where data live, defaults to the default device -The following example defines a view "temperature" which has dimensions columns and levels: -``` -Kokkos::View temperature("temperature", ncols, nlevs); +The following example defines a view "temperature" which has dimensions columns +and levels: + +```cpp +Kokkos::View temperature( + "temperature", ncols, nlevs); ``` ### Deep Copy -Kokkos provides `Kokkos::deep_copy(dst, src)` which copies data between views of the same dimensions, or a scalar values into a view. Common uses -``` +Kokkos provides `Kokkos::deep_copy(dst, src)` which copies data between views +of the same dimensions, or a scalar values into a view. Common uses + +```cpp Kokkos::deep_copy(view0, view1); // Copy all data from view1 into view0 Kokkos::deep_copy(view0, 5); // Set all values of view0 to 5 ``` -As seen in the next section, we can use `deep_copy()` to copy data between host and device. + +As seen in the next section, we can use `deep_copy()` to copy data between host +and device. ### Mirror Views -We will often need to have memory allocation the resides on device (for computation), and then need that identical data on host (say, for output). Kokkos has a concept of mirror views, where data can be copied from host to device and vice versa. +We will often need to have memory allocation the resides on device (for +computation), and then need that identical data on host (say, for output). +Kokkos has a concept of mirror views, where data can be copied from host to +device and vice versa. Here is an example using the device view `temperature` from above -``` -// Allocate view on host that exactly mirrors the size of layout of the device view + +```cpp +// Allocate view on host that exactly mirrors the size of layout of the device +view auto host_temperature = Kokkos::create_mirror_view(temperature); // Copy all data from device to host Kokkos::deep_copy(host_temperature, temperature); ``` + Kokkos also offers an all-in-one option -``` + +```cpp // Note: must hand the host device instance as first argument -auto host_temperature = Kokkos::create_mirror_view_and_copy(Kokkos::DefaultHostDevice(), temperature); +auto host_temperature = Kokkos::create_mirror_view_and_copy( + Kokkos::DefaultHostDevice(), temperature); ``` ### Parallel Execution -The most basic parallel execution pattern used by EAMxx is the `Kokkos::parallel_for` which defines a for-loop with completely independent iterations. The `parallel_for` takes in an optional label for debugging, an execution policy, which defines a range and location (host or device) for the code to be run, and a lambda describing the body of code to be executed. The following are execution policies used in EAMxx - - - `int count`: 1D iteration range `[0, count)` - - `RangePolicy(int beg, int end)`: 1D iteration range for indices `[beg, end)` - - `MDRangePolicy>(int[N] beg, int[N] end)`: multi-dimensional iteration range `[beg, end)` - - `TeamPolicy(int league_size, int team_size, int vector_size)`: 1D iteration over `league_size`, assigned to thread teams of size `team_size`, each with `vector_size` vector lanes. Both `team_size` and `vector_size` can be given `Kokkos::AUTO` as input for Kokkos to automatically compute. +The most basic parallel execution pattern used by EAMxx is the +`Kokkos::parallel_for` which defines a for-loop with completely independent +iterations. The `parallel_for` takes in an optional label for debugging, an +execution policy, which defines a range and location (host or device) for the +code to be run, and a lambda describing the body of code to be executed. The +following are execution policies used in EAMxx + +- `int count`: 1D iteration range `[0, count)` +- `RangePolicy(int beg, int end)`: 1D iteration range for indices + `[beg, end)` +- `MDRangePolicy>(int[N] beg, int[N] end)`: multi- + dimensional iteration range `[beg, end)` +- `TeamPolicy(int league_size, int team_size, int vector_size)`: 1D + iteration over `league_size`, assigned to thread teams of size `team_size`, + each with `vector_size` vector lanes. Both `team_size` and `vector_size` can + be given `Kokkos::AUTO` as input for Kokkos to automatically compute. If no `ExecSpace` template is given, the default execution space is used. -For lambda capture, use `KOKKOS_LAMBDA` macro which sets capture automatically based on architecture. +For lambda capture, use `KOKKOS_LAMBDA` macro which sets capture automatically +based on architecture. Example using `RangePolicy` to initialize a view -``` -Kokkos::View temperature("temperature", ncols, nlevs); + +```cpp +Kokkos::View temperature("temperature", ncols, + nlevs); Kokkos::parallel_for("Init_temp", - Kokkos::RangePolicy(0, ncols*nlevs), - KOKKOS_LAMBDA (const int idx) { + Kokkos::RangePolicy(0, ncols*nlevs), + KOKKOS_LAMBDA (const int idx) { int icol = idx/nlevs; int ilev = idx%nlevs; temperature(icol, ilev) = 0; }); ``` + Same example with `TeamPolicy` -``` + +```cpp Kokkos::parallel_for("Init_temp", - Kokkos::TeamPolicy(ncols*nlevs, Kokkos::AUTO, Kokkos::AUTO), - KOKKOS_LAMBDA (const TeamPolicy::member_type& team) { + Kokkos::TeamPolicy(ncols*nlevs, Kokkos::AUTO, Kokkos::AUTO), + KOKKOS_LAMBDA (const TeamPolicy::member_type& team) { // league_rank() gives the index for this team int icol = team.league_rank()/nlevs; int ilev = team.league_rank()%nlevs; @@ -105,32 +169,39 @@ Kokkos::parallel_for("Init_temp", ### Hierarchical Parallelism -Using `TeamPolicy`, we can have up to three nested levels of parallelism: team parallelism, thread parallelism, vector parallelism. These nested policies can be called within the lambda body using the following execution policies +Using `TeamPolicy`, we can have up to three nested levels of parallelism: team +parallelism, thread parallelism, vector parallelism. These nested policies can +be called within the lambda body using the following execution policies - - `TeamThreadRange(team, begin, end)`: execute over threads of a team - - `TeamVectorRange(team, begin, end)`: execute over threads and vector lanes of a team - - `ThreadVectorRange(team, begin, end)`: execute over vector lanes of a thread +- `TeamThreadRange(team, begin, end)`: execute over threads of a team +- `TeamVectorRange(team, begin, end)`: execute over threads and vector lanes of + a team +- `ThreadVectorRange(team, begin, end)`: execute over vector lanes of a thread An example of using these policies -``` + +```cpp Kokkos::View Q("tracers", ncols, ntracers, nlevs); Kokkos::parallel_for(Kokkos::TeamPolicy(ncols, Kokkos::AUTO), - KOKKOS_LAMBDA (TeamPolicy::member_type& team) { + KOKKOS_LAMBDA (TeamPolicy::member_type& team) { int icol = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlevs), [&](int ilev) { - temperature(icol, ilev) = 0; + temperature(icol, ilev) = 0; }); Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlevs), [&](int ilev) { - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, ntracers), [&](int iq) { - Q(icol, iq, ilev) = 0; - }); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, ntracers), [&](int iq) { + Q(icol, iq, ilev) = 0; + }); }); }); ``` -IMPORTANT! Nested policies cannot be used in arbitrary order. `ThreadVectorRange` must be used inside a `TeamThreadRange`, and `TeamVectorRange` must be the only level of nested parallelism. -``` +IMPORTANT! Nested policies cannot be used in arbitrary order. `ThreadVectorRange` +must be used inside a `TeamThreadRange`, and `TeamVectorRange` must be the only +level of nested parallelism. + +```cpp Kokkos::parallel_for(TeamPolicy(...), ... { // OK Kokkos::parallel_for(TeamThreadRange, ... { @@ -139,9 +210,9 @@ Kokkos::parallel_for(TeamPolicy(...), ... { // OK Kokkos::parallel_for(TeamThreadRange, ... { - Kokkos::parallel_for(ThreadVectorRange, ... { + Kokkos::parallel_for(ThreadVectorRange, ... { - }); + }); }); // OK @@ -156,13 +227,15 @@ Kokkos::parallel_for(TeamPolicy(...), ... { // WRONG, a TeamVectorRange must be the only nested level Kokkos::parallel_for(TeamVectorRange, ...{ - Kokkos::parallel_for(ThreadVectorRange, ... { + Kokkos::parallel_for(ThreadVectorRange, ... { - }); + }); }); }); ``` -Using these incorrectly can be very tricky to debug as the code almost certainly will _not_ error out, but race conditions will exist among threads. + +Using these incorrectly can be very tricky to debug as the code almost certainly +will _not_ error out, but race conditions will exist among threads. ## EKAT @@ -175,6 +248,3 @@ Using these incorrectly can be very tricky to debug as the code almost certainly ### Scratch Memory: WorspaceManager ### Algorithms - - - diff --git a/components/eamxx/docs/developer/managers.md b/components/eamxx/docs/developer/managers.md index 676449a2184..fa98c8b1d72 100644 --- a/components/eamxx/docs/developer/managers.md +++ b/components/eamxx/docs/developer/managers.md @@ -1 +1 @@ -## FieldManager and GridsManager +# FieldManager and GridsManager diff --git a/components/eamxx/docs/developer/processes.md b/components/eamxx/docs/developer/processes.md index 9ad556a3183..adb90e2dfbc 100644 --- a/components/eamxx/docs/developer/processes.md +++ b/components/eamxx/docs/developer/processes.md @@ -1,59 +1,77 @@ # Atmospheric Processes -In EAMxx, `AtmosphereProcess` (AP) is an abstract class representing a portion of the atmosphere timestep algorithm. -In simple terms, an AP is an object that given certain input fields performs some calculations to compute -some output fields. The concrete AP classes allow to create a buffer layer between particular packages (e.g., -dynamics dycore, physics parametrizations) and the atmosphere driver (AD), allowing separation of concerns, -so that the AD does not need to know details about the package, and the package does not need to know about -the EAMxx infrastructure. - -To enhance this separation of concerns, EAMxx implements two more classes for handling APs: - -- the concrete class `AtmosphereProcessGroup` (APG), which allows to group together a set of AP's, which can be seen from outside as a single process; -- the `AtmosphereProcessFactory` class, which allows an APG to create its internal processes without any knowledge of -what they are. - -This infrastructure allows the AD to view the whole atmosphere as a single APG, and to be completely agnostic to -what processes are run, and in which order. This design allows to have a code base that is cleaner, self-container, -and easy to test via a battery of targeted unit tests. - -In EAMxx, we already have a few concrete AP's, interfacing the AD to the Hommexx non-hydrostatic dycore as well as -some physics parametrizations (P3, SHOC, RRMTPG, etc). In the next section we describe the interfaces of an AP class, -and we show an example of how to write a new concrete AP class. +In EAMxx, `AtmosphereProcess` (AP) is an abstract class representing a portion +of the atmosphere timestep algorithm. In simple terms, an AP is an object that +given certain input fields performs some calculations to compute some output +fields. The concrete AP classes allow to create a buffer layer between +particular packages (e.g., dynamics dycore, physics parametrizations) and the +atmosphere driver (AD), allowing separation of concerns, so that the AD does +not need to know details about the package, and the package does not need to +know about the EAMxx infrastructure. + +To enhance this separation of concerns, EAMxx implements two more classes for +handling APs: + +- the concrete class `AtmosphereProcessGroup` (APG), which allows to group + together a set of AP's, which can be seen from outside as a single process; +- the `AtmosphereProcessFactory` class, which allows an APG to create its + internal processes without any knowledge of what they are. + +This infrastructure allows the AD to view the whole atmosphere as a single APG, +and to be completely agnostic to what processes are run, and in which order. +This design allows to have a code base that is cleaner, self-container, and +easy to test via a battery of targeted unit tests. + +In EAMxx, we already have a few concrete AP's, interfacing the AD to the +Hommexx non-hydrostatic dycore as well as some physics parametrizations (P3, +SHOC, RRMTPG, etc). In the next section we describe the interfaces of an AP +class, and we show an example of how to write a new concrete AP class. ## Atmosphere process interfaces An AP has several interfaces, which can be grouped into three categories: - - initialization: these interfaces are used to create the AP, as well as to initialize internal data structures; - - run: these interfaces are used to make the AP compute its output fields from its input fields; - - finalization: these interfaces are used to perform any clean up operation (e.g., release files) before the AP is - destroyed. - -Among the above, the initialization sequence is the most complex, and conists of several steps: - - - The AD creates the APG corresponding to the whole atmosphere. As mentioned above, this phase will make use of a factory, - which allows the AD to be agnostic to what is actually in the group. All AP's can start performing any initialization - work that they can, but at this point they are limited to use only an MPI communicator as well as a list of runtime - parameters (which were previously read from an input file). - - The AD passes a `GridsManager` to the AP's, so that they can get information about the grids they need. At this point, - all AP's have all the information they need to establish the layout of the input and output fields they need, - and can store a list of these "requests" - - After creating all fields (based on AP's requests), the AD passes a copy of each input and output field to - the AP's. These fields will be divided in "required" and "computed", which differ in that the former are only - passed to the AP's as 'read-only' fields (see the [field](field.md#Field) documentation for more details) - - The AP's are queried for how much scratch memory they may need at run time. After all AP's communicate their needs, - the AD will provide a pointer to scratch memory to the AP's. This is memory that can be used to initialize - temporary views/fields or other internal data structures. All AP's are given the same pointer, which means no - data persistence should be expected at run time between one timestep and the next. - - The AD calls the 'initialize' method on each AP. At this point, all fields are set, and AP's can complete any - remaining initialization task - -While the base AP class provides an (empty) implementation for some methods, in case derived classes do not need a -feature, some methods are purely virtual, and concrete classes will have to override them. Looking at existing -concrete AP implementations is a good way to have a first idea of what a new AP class needs to implement. Here, -we show go over the possible implementation of these methods in a hypothetical AP class. The header file may -look something like this +- initialization: these interfaces are used to create the AP, as well as to + initialize internal data structures; +- run: these interfaces are used to make the AP compute its output fields from + its input fields; +- finalization: these interfaces are used to perform any clean up operation + (e.g., release files) before the AP is destroyed. + +Among the above, the initialization sequence is the most complex, and consists +of several steps: + +- The AD creates the APG corresponding to the whole atmosphere. As mentioned + above, this phase will make use of a factory, which allows the AD to be + agnostic to what is actually in the group. All AP's can start performing any + initialization work that they can, but at this point they are limited to use + only an MPI communicator as well as a list of runtime parameters (which were + previously read from an input file). +- The AD passes a `GridsManager` to the AP's, so that they can get information + about the grids they need. At this point, all AP's have all the information + they need to establish the layout of the input and output fields they need, + and can store a list of these "requests" +- After creating all fields (based on AP's requests), the AD passes a copy of + each input and output field to the AP's. These fields will be divided in + "required" and "computed", which differ in that the former are only passed + to the AP's as 'read-only' fields (see the [field](field.md#Field) + documentation for more details) +- The AP's are queried for how much scratch memory they may need at run time. + After all AP's communicate their needs, the AD will provide a pointer to + scratch memory to the AP's. This is memory that can be used to initialize + temporary views/fields or other internal data structures. All AP's are given + the same pointer, which means no data persistence should be expected at run + time between one timestep and the next. +- The AD calls the 'initialize' method on each AP. At this point, all fields + are set, and AP's can complete any remaining initialization task + +While the base AP class provides an (empty) implementation for some methods, in +case derived classes do not need a feature, some methods are purely virtual, +and concrete classes will have to override them. Looking at existing concrete +AP implementations is a good way to have a first idea of what a new AP class +needs to implement. Here, we show go over the possible implementation of these +methods in a hypothetical AP class. The header file may look something like +this ```c++ #include @@ -86,21 +104,26 @@ protected: bool m_has_blah; }; ``` + A few comments: - - we added two views to the class, which are meant to be used to store intermediate results during calculations at -runtime; - - there are other methods that the class can override (such as additional operations when the AD sets a field in the - AP), but most AP's only need to override only these; - - we strongly encourage to add the keyword `override` when overriding a method; in case of small typos (e.g., missing - a `&` or a `const`, the compiler will be erroring out, since the signature will not match any virtual method in the - base class; - - `findalize_impl` is often empty; unless the AP is managing external resources, everything should be correctly released - during destruction; - - the two methods for buffers can be omitted if the AP does not need any scratch memory (and the default implementation - from the base class will be used). - -Here is a possible implementation of the methods, with some inline comments to explain +- we added two views to the class, which are meant to be used to store + intermediate results during calculations at runtime; +- there are other methods that the class can override (such as additional + operations when the AD sets a field in the AP), but most AP's only need to + override only these; +- we strongly encourage to add the keyword `override` when overriding a method; + in case of small typos (e.g., missing a `&` or a `const`, the compiler will + be erroring out, since the signature will not match any virtual method in the + base class; +- `finalize_impl` is often empty; unless the AP is managing external resources, + everything should be correctly released during destruction; +- the two methods for buffers can be omitted if the AP does not need any + scratch memory (and the default implementation from the base class will be + used). + +Here is a possible implementation of the methods, with some inline comments to +explain ```c++ MyProcess::MyProcess (const ekat::Comm& comm, const ekat::ParameterList& pl) diff --git a/components/eamxx/docs/developer/source_tree.md b/components/eamxx/docs/developer/source_tree.md index 15c018cc885..ed8270db635 100644 --- a/components/eamxx/docs/developer/source_tree.md +++ b/components/eamxx/docs/developer/source_tree.md @@ -56,4 +56,3 @@ You'll also see some other files in the `src/` directory itself, such as + `scream_config.h.in`: A template for generating a C++ header file with EAMxx configuration information. - diff --git a/components/eamxx/docs/developer/standalone_testing.md b/components/eamxx/docs/developer/standalone_testing.md index e2bb5d62556..633dcc34dc1 100644 --- a/components/eamxx/docs/developer/standalone_testing.md +++ b/components/eamxx/docs/developer/standalone_testing.md @@ -27,26 +27,30 @@ be made known to EAMxx by editing the eamxx/scripts/machines_specs.py files. There are some instructions on what to do at the top of this file. `test-all-scream` has a good help dump -``` -% cd $scream_repo/components/eamxx -% ./scripts/test-all-scream -h + +```shell +cd $scream_repo/components/eamxx +./scripts/test-all-scream -h ``` If you are unsure of the cmake configuration for you development cycle, one trick you can use is to run `test-all-scream` for the `dbg` test and just copy the cmake command it prints (then ctrl-C the process). -``` -% cd $scream_repo/components/eamxx -% ./scripts/test-all-scream -t dbg -m $machine -* wait for a few seconds* -* Ctrl-C * -* Copy the contents of DCMAKE_COMMAND that was passed to ctest * -* Add "cmake" to beginning of contents and path to eamxx at the end. * + +```shell +cd $scream_repo/components/eamxx +./scripts/test-all-scream -t dbg -m $machine +# wait for a few seconds* +# Ctrl-C * +# Copy the contents of DCMAKE_COMMAND that was passed to ctest * +# Add "cmake" to beginning of contents and path to eamxx at the end. * ``` Considerations for using `test-all-scream`: + * Your machine must be known to our scripts, see above. -* If you try to run commands by-hand (outside of test-all-scream; cmake, make, ctest, etc), you'll need to remember to +* If you try to run commands by-hand (outside of test-all-scream; + cmake, make, ctest, etc), you'll need to remember to load the scream-env into your shell, which can be done like this: `cd eamxx/scripts; eval $(./scripts/scream-env-cmd $machine)` * test-all-scream expects to be run from a compute node if you @@ -63,7 +67,7 @@ Considerations for using `test-all-scream`: Before running the tests, generate a baseline file: -``` +```shell cd $RUN_ROOT_DIR make baseline ``` @@ -75,7 +79,7 @@ path has been provided. To run all of SCREAM's tests, make sure you're in `$RUN_ROOT_DIR` and type -``` +```shell ctest -VV ``` @@ -84,7 +88,7 @@ This runs everything and reports results in an extra-verbose (`-VV`) manner. You can also run subsets of the SCREAM tests. For example, to run only the P3 regression tests (again, from the `$RUN_ROOT_DIR` directory), use -``` +```shell ctest -R p3_regression ``` @@ -94,13 +98,13 @@ We can create groupings of tests by using **labels**. For example, we have a `driver` label that runs tests for SCREAM's standalone driver. You can see a list of available labels by typing -``` +```shell ctest --print-labels ``` To see which tests are associated with a given label (e.g. `driver`), use -``` +```shell ctest -L driver -N ``` @@ -117,4 +121,3 @@ on the C++/Kokkos implementation, you can invoke any new tests to the function If the reference Fortran implementation changes enough that a new baseline file is required, make sure to let other SCREAM team members know, in order to minimize disruptions. - diff --git a/components/eamxx/docs/developer/style_guide.md b/components/eamxx/docs/developer/style_guide.md index f4367833009..4f6f340cb66 100644 --- a/components/eamxx/docs/developer/style_guide.md +++ b/components/eamxx/docs/developer/style_guide.md @@ -7,4 +7,3 @@ Here's our style guide. Let the holy wars begin! ## Functions and Methods ## Variables - From fa86f7a57e9a6cc8aba91aea1d53b20b86772165 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 9 Nov 2024 15:46:42 -0800 Subject: [PATCH 271/366] EAMxx: Adds a SMS ne30 test that uses ne4 emission file --- .../scream/mam4xx/force_remap/shell_commands | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/force_remap/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/force_remap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/force_remap/shell_commands new file mode 100644 index 00000000000..078b8e93a25 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/force_remap/shell_commands @@ -0,0 +1,26 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +#Update IC file and add drydep process +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b + +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_so2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_so4_a1_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_so4_a2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_pom_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_bc_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_num_a1_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_num_a2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_num_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_soag_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::aero_microphys_remap_file='/qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/nco_scripts_mapping_files/map_ne4pg2_to_ne30pg2_nco_c20241107.nc' -b + + From 9f6e83e422f5b5cb0969744d248f4086e570bdc0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 9 Nov 2024 15:57:05 -0800 Subject: [PATCH 272/366] EAMxx:Modified folder name nad uses alias for atmchange --- .../scream/mam4xx/force_remap/shell_commands | 26 ----------------- .../remap_emiss_ne4_ne30/shell_commands | 28 +++++++++++++++++++ 2 files changed, 28 insertions(+), 26 deletions(-) delete mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/force_remap/shell_commands create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/force_remap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/force_remap/shell_commands deleted file mode 100644 index 078b8e93a25..00000000000 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/force_remap/shell_commands +++ /dev/null @@ -1,26 +0,0 @@ - -#!/bin/sh -#------------------------------------------------------ -# MAM4xx adds additionaltracers to the simulation -# Increase number of tracers for MAM4xx simulations -#------------------------------------------------------ - -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b - -#------------------------------------------------------ -#Update IC file and add drydep process -#------------------------------------------------------ -$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b - -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_so2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_so4_a1_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_so4_a2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_pom_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_bc_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_num_a1_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_num_a2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_num_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::mam4_soag_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mam4_aero_microphys::aero_microphys_remap_file='/qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/nco_scripts_mapping_files/map_ne4pg2_to_ne30pg2_nco_c20241107.nc' -b - - diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands new file mode 100644 index 00000000000..538fbcf1b18 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands @@ -0,0 +1,28 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +#Update IC file and add drydep process +#------------------------------------------------------ +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +ATMCHANGE physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b + +ATMCHANGE mam4_aero_microphys::mam4_so2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_so4_a1_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_so4_a2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_pom_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_bc_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a1_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_soag_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::aero_microphys_remap_file='/qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/nco_scripts_mapping_files/map_ne4pg2_to_ne30pg2_nco_c20241107.nc' -b + + From c07bc7283ada4535cba29ab068c8e4ce0d253669 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 9 Nov 2024 16:41:32 -0800 Subject: [PATCH 273/366] EAMxx:Adds an alias and upload the mapping file ne4->ne30 --- .../scream/mam4xx/remap_emiss_ne4_ne30/shell_commands | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands index 538fbcf1b18..da79db14fbe 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands @@ -10,7 +10,7 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/u #------------------------------------------------------ #Update IC file and add drydep process #------------------------------------------------------ -ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange +alias ATMCHANGE='$CIMEROOT/../components/eamxx/scripts/atmchange' ATMCHANGE physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b @@ -23,6 +23,6 @@ ATMCHANGE mam4_aero_microphys::mam4_num_a1_verti_emiss_file_name='${DIN_LOC_ROOT ATMCHANGE mam4_aero_microphys::mam4_num_a2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b ATMCHANGE mam4_aero_microphys::mam4_num_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b ATMCHANGE mam4_aero_microphys::mam4_soag_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -ATMCHANGE mam4_aero_microphys::aero_microphys_remap_file='/qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/nco_scripts_mapping_files/map_ne4pg2_to_ne30pg2_nco_c20241107.nc' -b +ATMCHANGE mam4_aero_microphys::aero_microphys_remap_file='${DIN_LOC_ROOT}/atm/scream/maps/map_ne4pg2_to_ne30pg2_nco_c20241108.nc' -b From e3f671730e16fdc706bfeb74057951ea21ab3473 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 9 Nov 2024 16:44:37 -0800 Subject: [PATCH 274/366] EAMxx: Adds remap test to nightlies --- cime_config/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/tests.py b/cime_config/tests.py index 3c89e8e9099..41270f6217c 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -757,6 +757,7 @@ "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-wetscav", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-drydep", + "SMS_D_Ln5.ne30pg2_oECv3.F2010-SCREAMv1-MPASSI.scream-mam4xx-remap_emiss_ne4_ne30" ) }, From 2508f4746571dd9c3aa0204b3deec79773c830cf Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 9 Nov 2024 16:46:03 -0800 Subject: [PATCH 275/366] EAMxx: Removes init file from mam4xx test as it is now picked automatically --- .../testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands index 1d6757a5bd9..56390e2c9b0 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands @@ -10,7 +10,6 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/u #------------------------------------------------------ #Update IC file and add drydep process #------------------------------------------------------ -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b From 806a6f1e928750464e419d3580dbb6e84627bf99 Mon Sep 17 00:00:00 2001 From: singhbalwinder Date: Sat, 9 Nov 2024 17:28:22 -0800 Subject: [PATCH 276/366] EAMxx: Fix a comment in the shell script --- .../scream/mam4xx/remap_emiss_ne4_ne30/shell_commands | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands index da79db14fbe..aa3affac25b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands @@ -7,9 +7,9 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b -#------------------------------------------------------ -#Update IC file and add drydep process -#------------------------------------------------------ +#---------------------------------------------------------------------------------------------------------- +# Add aerosol microphysics process, force ne4pg2 emissions files and provide a ne4pg2->ne30pg2 mapping file +#---------------------------------------------------------------------------------------------------------- alias ATMCHANGE='$CIMEROOT/../components/eamxx/scripts/atmchange' ATMCHANGE physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b From 9788c5718aeca16bcdb2026de1df26e4dfe96303 Mon Sep 17 00:00:00 2001 From: singhbalwinder Date: Sat, 9 Nov 2024 17:29:28 -0800 Subject: [PATCH 277/366] EAMxx: Fixes a comment in the microphysics testmod file --- .../scream/mam4xx/aero_microphysics/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands index 56390e2c9b0..ac1709f7dca 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands @@ -8,7 +8,7 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ -#Update IC file and add drydep process +# Add microphysics process #------------------------------------------------------ $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b From 97b2dd76acc65627078a99da736000841876290e Mon Sep 17 00:00:00 2001 From: singhbalwinder Date: Sat, 9 Nov 2024 17:31:26 -0800 Subject: [PATCH 278/366] EAMxx: Fixes a comment in the newly added test --- .../scream/mam4xx/remap_emiss_ne4_ne30/shell_commands | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands index aa3affac25b..606ca463f26 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands @@ -7,9 +7,11 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b -#---------------------------------------------------------------------------------------------------------- -# Add aerosol microphysics process, force ne4pg2 emissions files and provide a ne4pg2->ne30pg2 mapping file -#---------------------------------------------------------------------------------------------------------- +#------------------------------------------------------ +# Add aerosol microphysics process, force ne4pg2 +# emission files and provide a ne4pg2->ne30pg2 mapping +# file +#------------------------------------------------------ alias ATMCHANGE='$CIMEROOT/../components/eamxx/scripts/atmchange' ATMCHANGE physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b From e95224ebe403de0edfde6f316efb82405aa0a8a7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 11 Nov 2024 09:26:14 -0700 Subject: [PATCH 279/366] Workflows: fix logic to execute/skip eamxx testing workflows --- .github/workflows/eamxx-sa-testing.yml | 50 +++++++++++++---------- .github/workflows/eamxx-scripts-tests.yml | 3 ++ .github/workflows/eamxx-v1-testing.yml | 23 ++++++----- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index bbfa12b80ac..baa1508ea6c 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -83,18 +83,21 @@ jobs: gcc-openmp: needs: [pre_process_pr] if: | - success() && github.event_name == 'schedule' || + !failure() && !cancelled() && ( - github.event_name == 'pull_request' && - needs.pre_process_pr.outputs.relevant_paths=='true' && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip openmp') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') - ) || ( - github.event_name == 'workflow_dispatch' && - github.event.inputs.job_to_run == 'gcc-openmp' || - github.event.inputs.job_to_run == 'all' + github.event_name == 'schedule' || + ( + github.event_name == 'pull_request' && + needs.pre_process_pr.outputs.relevant_paths=='true' && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip openmp') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') + ) || ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'gcc-openmp' || + github.event.inputs.job_to_run == 'all' + ) ) runs-on: [self-hosted, ghci-snl-cpu, gcc] strategy: @@ -130,18 +133,21 @@ jobs: gcc-cuda: needs: [pre_process_pr] if: | - success() && github.event_name == 'schedule' || + !failure() && !cancelled() && ( - github.event_name == 'pull_request' && - needs.pre_process_pr.outputs.relevant_paths=='true' && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip cuda') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') - ) || ( - github.event_name == 'workflow_dispatch' && - github.event.inputs.job_to_run == 'gcc-cuda' || - github.event.inputs.job_to_run == 'all' + github.event_name == 'schedule' || + ( + github.event_name == 'pull_request' && + needs.pre_process_pr.outputs.relevant_paths=='true' && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip cuda') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') + ) || ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'gcc-cuda' || + github.event.inputs.job_to_run == 'all' + ) ) runs-on: [self-hosted, ghci-snl-cuda, cuda, gcc] strategy: diff --git a/.github/workflows/eamxx-scripts-tests.yml b/.github/workflows/eamxx-scripts-tests.yml index 6d39200e273..8de453de387 100644 --- a/.github/workflows/eamxx-scripts-tests.yml +++ b/.github/workflows/eamxx-scripts-tests.yml @@ -62,11 +62,14 @@ jobs: cpu-gcc: needs: [pre_process_pr] if: | + !failure() && !cancelled() && + ( github.event_name != 'pull_request' || ( needs.pre_process_pr.outputs.relevant_paths == 'true' && !contains(needs.pre_process_pr.outputs.labels, 'CI: skip eamxx-all') ) + ) runs-on: [self-hosted, gcc, ghci-snl-cpu] steps: - name: Check out the repository diff --git a/.github/workflows/eamxx-v1-testing.yml b/.github/workflows/eamxx-v1-testing.yml index 3a4b28cfed3..43cc7623f88 100644 --- a/.github/workflows/eamxx-v1-testing.yml +++ b/.github/workflows/eamxx-v1-testing.yml @@ -75,17 +75,20 @@ jobs: cpu-gcc: needs: [pre_process_pr] if: | - github.event_name == 'schedule' || + !failure() && !cancelled() && ( - github.event_name == 'pull_request' && - needs.pre_process_pr.outputs.relevant_paths=='true' && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-v1') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') - ) || ( - github.event_name == 'workflow_dispatch' && - github.event.inputs.job_to_run == 'cpu-gcc' || - github.event.inputs.job_to_run == 'all' + github.event_name == 'schedule' || + ( + github.event_name == 'pull_request' && + needs.pre_process_pr.outputs.relevant_paths=='true' && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-v1') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') + ) || ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'cpu-gcc' || + github.event.inputs.job_to_run == 'all' + ) ) runs-on: [self-hosted, gcc, ghci-snl-cpu] strategy: From 37ae73afd8bc8f92c70946bdfd9b3d8f7ccaa309 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Nov 2024 10:22:52 -0700 Subject: [PATCH 280/366] EMAxx - Update comment in enum TracerFileType to avoid confusion. --- .../physics/mam/readfiles/tracer_reader_utils.hpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp index bec628b1f96..c5d573e06ff 100644 --- a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp @@ -66,13 +66,18 @@ struct ForcingHelper { }; enum TracerFileType { - // file with PS ncol, lev, and time + // file with ncol, lev, ilev, time and has P0 and PS as variables + // example: oxidants FORMULA_PS, - // nc zonal file from ncremap - ZONAL, // vertical emission files + // example: linoz + ZONAL, + // file with ncol, altitude, altitude_int, time + // example: elevated (at a height) emissions of aerosols and precursors + // NOTE: we must rename the default vertical tags when horiz remapping + // NOTE: we vert remap in a different routine in mam4xx VERT_EMISSION, - // + // Placeholder for cases where no file type is applicable NONE }; From 9331f79677470fba68798eec35029856d436a3e8 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Mon, 11 Nov 2024 12:06:03 -0600 Subject: [PATCH 281/366] Adding support for ADIOS builds with BLOSC Adding support for ADIOS builds that include support for BLOSC (lossless compression) in FindPIO cmake module --- components/cmake/modules/FindPIO.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/cmake/modules/FindPIO.cmake b/components/cmake/modules/FindPIO.cmake index 0277918ac8e..5589dff0ed0 100644 --- a/components/cmake/modules/FindPIO.cmake +++ b/components/cmake/modules/FindPIO.cmake @@ -42,6 +42,9 @@ endif() # we can assume that an MPI case with ADIOS2_ROOT set is probably # using adios. if (NOT MPILIB STREQUAL "mpi-serial" AND DEFINED ENV{ADIOS2_ROOT}) + if(DEFINED ENV{BLOSC2_ROOT}) + set(ENV{Blosc2_DIR} "$ENV{BLOSC2_ROOT}") + endif() find_package(MPI REQUIRED COMPONENTS C) find_package(ADIOS2 REQUIRED COMPONENTS C) list(APPEND PIOLIBS adios2::adios2) From e779856aeff251199442d694d124073c876b7dde Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 11 Nov 2024 11:28:52 -0700 Subject: [PATCH 282/366] Update based on better ekat flag handling --- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 8 ++--- .../p3/tests/infra/p3_unit_tests_common.hpp | 31 ++++++------------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 784d3436c88..129e3c455e2 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -59,7 +59,7 @@ endif() CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" LIBS p3 p3_test_infra - EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" + EXE_ARGS "--args ${BASELINE_FILE_ARG}" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3;physics") @@ -68,7 +68,7 @@ CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" if (SCREAM_ENABLE_BASELINE_TESTS) CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp LIBS p3 p3_test_infra - EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" + EXE_ARGS "--args ${BASELINE_FILE_ARG}" COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF LABELS "p3;physics;fail" ${FORCE_RUN_DIFF_FAILS}) @@ -77,7 +77,7 @@ endif() if (NOT SCREAM_P3_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" LIBS p3_sk p3_test_infra - EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" + EXE_ARGS "--args ${BASELINE_FILE_ARG}" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3_sk;physics") @@ -86,7 +86,7 @@ if (NOT SCREAM_P3_SMALL_KERNELS) if (SCREAM_ENABLE_BASELINE_TESTS) CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp LIBS p3_sk p3_test_infra - EXE_ARGS "--flags=\\'${BASELINE_FILE_ARG}\\'" + EXE_ARGS "--args ${BASELINE_FILE_ARG}" COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF LABELS "p3_sk;physics;fail" ${FORCE_RUN_DIFF_FAILS}) diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp index 523315ae5ec..9dd7dee95b3 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp @@ -85,28 +85,17 @@ struct UnitWrap { { Functions::p3_init(); // many tests will need fortran table data auto& ts = ekat::TestSession::get(); - auto raw_flags = ts.flags.begin()->first; - std::stringstream ss(raw_flags); - std::string flag; - bool next_token_is_path = false; - while (ss >> flag) { - if (flag == "-c") { - m_baseline_action = COMPARE; - } - else if (flag == "-g") { - m_baseline_action = GENERATE; - } - else if (flag == "-n") { - m_baseline_action = NONE; - } - else if (flag == "-b") { - next_token_is_path = true; - } - else if (next_token_is_path) { - m_baseline_path = flag; - next_token_is_path = false; - } + if (ts.flags["c"]) { + m_baseline_action = COMPARE; + } + else if (ts.flags["g"]) { + m_baseline_action = GENERATE; } + else if (ts.flags["n"]) { + m_baseline_action = NONE; + } + m_baseline_path = ts.params["b"]; + EKAT_REQUIRE_MSG( !(m_baseline_action != NONE && m_baseline_path == ""), "P3 unit test flags problem: baseline actions were requested but no baseline path was provided"); From 1062373c639fdf3a5c8860f9bf80e1266b3b2986 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Tue, 5 Nov 2024 14:38:08 -0800 Subject: [PATCH 283/366] Expose EAMxx to the git-version hash saved in the component cpl. This commit retrieves the git-hash saved in the component coupler and makes sure it is passed into the atmosphere driver. This is then used to improve the netcdf metadata in all EAMxx output files. --- components/eamxx/src/control/atmosphere_driver.cpp | 7 +++++-- components/eamxx/src/control/atmosphere_driver.hpp | 3 ++- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 10 ++++++---- .../src/mct_coupling/scream_cxx_f90_interface.cpp | 5 +++-- components/eamxx/src/mct_coupling/scream_f2c_mod.F90 | 6 ++++-- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 9cf2b48e358..0f1cb1e31ab 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -807,7 +807,8 @@ void AtmosphereDriver:: set_provenance_data (std::string caseid, std::string rest_caseid, std::string hostname, - std::string username) + std::string username, + std::string versionid) { #ifdef SCREAM_CIME_BUILD // Check the inputs are valid @@ -816,6 +817,7 @@ set_provenance_data (std::string caseid, "Error! Invalid restart case id: " + rest_caseid + "\n"); EKAT_REQUIRE_MSG (hostname!="", "Error! Invalid hostname: " + hostname + "\n"); EKAT_REQUIRE_MSG (username!="", "Error! Invalid username: " + username + "\n"); + EKAT_REQUIRE_MSG (versionid!="", "Error! Invalid version: " + versionid + "\n"); #else caseid = rest_caseid = m_casename; char* user = new char[32]; @@ -835,13 +837,14 @@ set_provenance_data (std::string caseid, } delete[] user; delete[] host; + versionid = EAMXX_GIT_VERSION; #endif auto& provenance = m_atm_params.sublist("provenance"); provenance.set("caseid",caseid); provenance.set("rest_caseid",rest_caseid); provenance.set("hostname",hostname); provenance.set("username",username); - provenance.set("version",std::string(EAMXX_GIT_VERSION)); + provenance.set("git_version",versionid); } void AtmosphereDriver:: diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index 41801745ea2..a3acfba5d94 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -116,7 +116,8 @@ class AtmosphereDriver void set_provenance_data (std::string caseid = "", std::string rest_caseid = "", std::string hostname = "", - std::string username = ""); + std::string username = "", + std::string versionid = ""); // Load initial conditions for atm inputs void initialize_fields (); diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index eb9165c2acb..0a248a2b608 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -137,8 +137,8 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! TODO: read this from the namelist? character(len=256) :: yaml_fname = "./data/scream_input.yaml" character(kind=c_char,len=256), target :: yaml_fname_c, atm_log_fname_c - character(len=256) :: caseid, username, hostname, rest_caseid - character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c, rest_caseid_c + character(len=256) :: caseid, username, hostname, rest_caseid, versionid + character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c, rest_caseid_c, versionid_c integer (kind=c_int) :: run_type_c !------------------------------------------------------------------------------- @@ -151,7 +151,8 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) infodata=infodata) call seq_infodata_getData(infodata, atm_phase=phase, start_type=run_type, & username=username, case_name=caseid, & - rest_case_name=rest_caseid, hostname=hostname) + rest_case_name=rest_caseid, hostname=hostname, & + model_version=versionid) call seq_infodata_PutData(infodata, atm_aero=.true.) call seq_infodata_PutData(infodata, atm_prognostic=.true.) @@ -213,10 +214,11 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call string_f2c(trim(rest_caseid),rest_caseid_c) call string_f2c(trim(hostname),hostname_c) call string_f2c(trim(username),username_c) + call string_f2c(trim(versionid),versionid_c) call scream_create_atm_instance (mpicom_atm, ATM_ID, yaml_fname_c, atm_log_fname_c, run_type_c, & INT(cur_ymd,kind=C_INT), INT(cur_tod,kind=C_INT), & INT(case_start_ymd,kind=C_INT), INT(case_start_tod,kind=C_INT), & - calendar_c, caseid_c, rest_caseid_c, hostname_c, username_c) + calendar_c, caseid_c, rest_caseid_c, hostname_c, username_c, versionid_c) ! Init MCT gsMap call atm_Set_gsMap_mct (mpicom_atm, ATM_ID, gsMap_atm) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 1e51aa47a36..8d2edd5ee9f 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -110,7 +110,8 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, const char* caseid, const char* rest_caseid, const char* hostname, - const char* username) + const char* username, + const char* versionid) { using namespace scream; using namespace scream::control; @@ -176,7 +177,7 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, ad.set_params(scream_params); ad.init_scorpio(atm_id); ad.init_time_stamps(run_t0,case_t0,run_type); - ad.set_provenance_data (caseid,rest_caseid,hostname,username); + ad.set_provenance_data (caseid,rest_caseid,hostname,username,versionid); ad.create_output_managers (); ad.create_atm_processes (); ad.create_grids (); diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 5adb0825041..434639e8f82 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -18,7 +18,8 @@ subroutine scream_create_atm_instance (f_comm,atm_id,yaml_fname,atm_log_fname, & run_start_ymd,run_start_tod, & case_start_ymd,case_start_tod, & calendar_name, & - caseid, rest_caseid, hostname, username) bind(c) + caseid, rest_caseid, hostname, username, & + versionid) bind(c) use iso_c_binding, only: c_int, c_char ! ! Input(s) @@ -27,7 +28,8 @@ subroutine scream_create_atm_instance (f_comm,atm_id,yaml_fname,atm_log_fname, & integer (kind=c_int), value, intent(in) :: run_start_tod, run_start_ymd integer (kind=c_int), value, intent(in) :: case_start_tod, case_start_ymd character(kind=c_char), target, intent(in) :: yaml_fname(*), atm_log_fname(*), calendar_name(*), & - caseid(*), rest_caseid(*), hostname(*), username(*) + caseid(*), rest_caseid(*), hostname(*), username(*), & + versionid(*) end subroutine scream_create_atm_instance subroutine scream_get_cols_latlon (lat, lon) bind(c) From 342958c701c7a53e17f022b06de3d85db201b8e0 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Nov 2024 11:35:30 -0700 Subject: [PATCH 284/366] EAMxx - Rename enum VERT_EMISSION to ELEVATED_EMISSIONS. --- .../eamxx_mam_microphysics_process_interface.cpp | 6 +++--- .../eamxx_mam_microphysics_process_interface.hpp | 2 +- .../mam/readfiles/tracer_reader_utils.hpp | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 6f26646ad6f..cc2cbbb2a6d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -326,10 +326,10 @@ void MAMMicrophysics::set_grids( ++i; } // end i EKAT_REQUIRE_MSG( - offset_emis_ver <= int(mam_coupling::MAX_NUM_VERT_EMISSION_FIELDS), + offset_emis_ver <= int(mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS), "Error! Number of fields is bigger than " - "MAX_NUM_VERT_EMISSION_FIELDS. Increase the " - "MAX_NUM_VERT_EMISSION_FIELDS in tracer_reader_utils.hpp \n"); + "MAX_NUM_ELEVATED_EMISSIONS_FIELDS. Increase the " + "MAX_NUM_ELEVATED_EMISSIONS_FIELDS in tracer_reader_utils.hpp \n"); } // Tracer external forcing data } // set_grids diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp index 6b1dd33dfaa..5ecd8c62461 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -232,7 +232,7 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { std::vector vert_emis_data_; std::map vert_emis_file_name_; std::map> vert_emis_var_names_; - view_2d vert_emis_output_[mam_coupling::MAX_NUM_VERT_EMISSION_FIELDS]; + view_2d vert_emis_output_[mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS]; view_3d extfrc_; mam_coupling::ForcingHelper forcings_[mam4::gas_chemistry::extcnt]; diff --git a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp index c5d573e06ff..c605d3a0b9f 100644 --- a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp @@ -69,14 +69,14 @@ enum TracerFileType { // file with ncol, lev, ilev, time and has P0 and PS as variables // example: oxidants FORMULA_PS, - // vertical emission files + // file with ncol, lev, ilev, time // example: linoz ZONAL, // file with ncol, altitude, altitude_int, time // example: elevated (at a height) emissions of aerosols and precursors // NOTE: we must rename the default vertical tags when horiz remapping // NOTE: we vert remap in a different routine in mam4xx - VERT_EMISSION, + ELEVATED_EMISSIONS, // Placeholder for cases where no file type is applicable NONE }; @@ -88,7 +88,7 @@ enum TracerDataIndex { BEG = 0, END = 1, OUT = 2 }; Therefore, if a file contains more than this number, it is acceptable to increase this limit. Currently, Linoz files have 8 fields. */ constexpr int MAX_NVARS_TRACER = 10; -constexpr int MAX_NUM_VERT_EMISSION_FIELDS = 25; +constexpr int MAX_NUM_ELEVATED_EMISSIONS_FIELDS = 25; // Linoz structures to help manage all of the variables: struct TracerTimeState { @@ -307,7 +307,7 @@ inline void setup_tracer_data(TracerData &tracer_data, // out // This type of files use altitude (zi) for vertical interpolation if(has_altitude) { nlevs_data = scorpio::get_dimlen(trace_data_file, "altitude"); - tracer_file_type = VERT_EMISSION; + tracer_file_type = ELEVATED_EMISSIONS; } EKAT_REQUIRE_MSG( nlevs_data != -1, @@ -339,7 +339,7 @@ inline void setup_tracer_data(TracerData &tracer_data, // out tracer_data.zonal_levs_ = levs; } - if(tracer_file_type == VERT_EMISSION) { + if(tracer_file_type == ELEVATED_EMISSIONS) { const int nilevs_data = scorpio::get_dimlen(trace_data_file, "altitude_int"); view_1d_host altitude_int_host("altitude_int_host", nilevs_data); @@ -448,7 +448,7 @@ inline std::shared_ptr create_tracer_data_reader( io_fields.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); - if(file_type == VERT_EMISSION ){ + if(file_type == ELEVATED_EMISSIONS ){ // NOTE: If we are using a vertical emission nc file with altitude instead of lev, // we must rename this tag. // We need to perform a shallow clone of io_grid because tags are const in this object. @@ -672,7 +672,7 @@ inline void perform_vertical_interpolation(const const_view_1d &altitude_int, const TracerData &input, const view_2d output[]) { EKAT_REQUIRE_MSG( - input.file_type == VERT_EMISSION, + input.file_type == ELEVATED_EMISSIONS, "Error! vertical interpolation only with altitude variable. \n"); const int ncols = input.ncol_; const int num_vars = input.nvars_; @@ -749,7 +749,7 @@ inline void advance_tracer_data( if(data_tracer.file_type == FORMULA_PS || data_tracer.file_type == ZONAL) { perform_vertical_interpolation(data_tracer.p_src_, p_tgt, data_tracer, output); - } else if(data_tracer.file_type == VERT_EMISSION) { + } else if(data_tracer.file_type == ELEVATED_EMISSIONS) { perform_vertical_interpolation(data_tracer.altitude_int_, zi_tgt, data_tracer, output); } From 3eb6d04c9ec31ab128a5d3544f636b5a602e0f4a Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Nov 2024 11:53:05 -0700 Subject: [PATCH 285/366] EAMxx - Rename variables: 'elevated_' prefix replaces 'vert_'. --- ...mxx_mam_microphysics_process_interface.cpp | 76 +++++++++---------- ...mxx_mam_microphysics_process_interface.hpp | 14 ++-- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index cc2cbbb2a6d..39734fadf89 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -260,67 +260,67 @@ void MAMMicrophysics::set_grids( for(const auto &var_name : extfrc_lst_) { std::string item_name = "mam4_" + var_name + "_verti_emiss_file_name"; const auto file_name = m_params.get(item_name); - vert_emis_file_name_[var_name] = file_name; + elevated_emis_file_name_[var_name] = file_name; } - vert_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; - vert_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; - vert_emis_var_names_["so4_a2"] = {"contvolc"}; - vert_emis_var_names_["pom_a4"] = {"BB"}; - vert_emis_var_names_["bc_a4"] = {"BB"}; - vert_emis_var_names_["num_a1"] = { + elevated_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; + elevated_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; + elevated_emis_var_names_["so4_a2"] = {"contvolc"}; + elevated_emis_var_names_["pom_a4"] = {"BB"}; + elevated_emis_var_names_["bc_a4"] = {"BB"}; + elevated_emis_var_names_["num_a1"] = { "num_a1_SO4_ELEV_BB", "num_a1_SO4_ELEV_ENE", "num_a1_SO4_ELEV_IND", "num_a1_SO4_ELEV_contvolc"}; - vert_emis_var_names_["num_a2"] = {"num_a2_SO4_ELEV_contvolc"}; + elevated_emis_var_names_["num_a2"] = {"num_a2_SO4_ELEV_contvolc"}; // num_a4 // FIXME: why the sectors in this files are num_a1; // I guess this should be num_a4? Is this a bug in the orginal nc files? - vert_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", + elevated_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", "num_a1_POM_ELEV_BB"}; - vert_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; + elevated_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; - int verti_emiss_cyclical_ymd = m_params.get("verti_emiss_ymd"); + int elevated_emiss_cyclical_ymd = m_params.get("verti_emiss_ymd"); for(const auto &var_name : extfrc_lst_) { - const auto file_name = vert_emis_file_name_[var_name]; - const auto var_names = vert_emis_var_names_[var_name]; + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; scream::mam_coupling::TracerData data_tracer; scream::mam_coupling::setup_tracer_data(data_tracer, file_name, - verti_emiss_cyclical_ymd); + elevated_emiss_cyclical_ymd); auto hor_rem = scream::mam_coupling::create_horiz_remapper( grid_, file_name, extfrc_map_file, var_names, data_tracer); auto file_reader = scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name, data_tracer.file_type); - VertEmissionsHorizInterp_.push_back(hor_rem); - VertEmissionsDataReader_.push_back(file_reader); - vert_emis_data_.push_back(data_tracer); - } // var_name vert emissions + ElevatedEmissionsHorizInterp_.push_back(hor_rem); + ElevatedEmissionsDataReader_.push_back(file_reader); + elevated_emis_data_.push_back(data_tracer); + } // var_name elevated emissions int i = 0; int offset_emis_ver = 0; for(const auto &var_name : extfrc_lst_) { - const auto file_name = vert_emis_file_name_[var_name]; - const auto var_names = vert_emis_var_names_[var_name]; + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; const int nvars = static_cast(var_names.size()); forcings_[i].nsectors = nvars; // I am assuming the order of species in extfrc_lst_. // Indexing in mam4xx is fortran. forcings_[i].frc_ndx = i + 1; - const auto io_grid_emis = VertEmissionsHorizInterp_[i]->get_tgt_grid(); + const auto io_grid_emis = ElevatedEmissionsHorizInterp_[i]->get_tgt_grid(); const int num_cols_io_emis = io_grid_emis->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_emis = io_grid_emis ->get_num_vertical_levels(); // Number of levels per column - vert_emis_data_[i].init(num_cols_io_emis, num_levs_io_emis, nvars); - vert_emis_data_[i].allocate_temporal_views(); - forcings_[i].file_alt_data = vert_emis_data_[i].has_altitude_; + elevated_emis_data_[i].init(num_cols_io_emis, num_levs_io_emis, nvars); + elevated_emis_data_[i].allocate_temporal_views(); + forcings_[i].file_alt_data = elevated_emis_data_[i].has_altitude_; for(int isp = 0; isp < nvars; ++isp) { forcings_[i].offset = offset_emis_ver; - vert_emis_output_[isp + offset_emis_ver] = - view_2d("vert_emis_output_", ncol_, nlev_); + elevated_emis_output_[isp + offset_emis_ver] = + view_2d("elevated_emis_output_", ncol_, nlev_); } offset_emis_ver += nvars; ++i; @@ -520,8 +520,8 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { for(int i = 0; i < static_cast(extfrc_lst_.size()); ++i) { scream::mam_coupling::update_tracer_data_from_file( - VertEmissionsDataReader_[i], curr_month, *VertEmissionsHorizInterp_[i], - vert_emis_data_[i]); + ElevatedEmissionsDataReader_[i], curr_month, *ElevatedEmissionsHorizInterp_[i], + elevated_emis_data_[i]); } invariants_ = view_3d("invarians", ncol_, nlev_, mam4::gas_chemistry::nfs); @@ -618,20 +618,20 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_output); // out Kokkos::fence(); - vert_emiss_time_state_.t_now = ts.frac_of_year_in_days(); + elevated_emiss_time_state_.t_now = ts.frac_of_year_in_days(); int i = 0; for(const auto &var_name : extfrc_lst_) { - const auto file_name = vert_emis_file_name_[var_name]; - const auto var_names = vert_emis_var_names_[var_name]; + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; const int nsectors = int(var_names.size()); - view_2d vert_emis_output[nsectors]; + view_2d elevated_emis_output[nsectors]; for(int isp = 0; isp < nsectors; ++isp) { - vert_emis_output[isp] = vert_emis_output_[isp + forcings_[i].offset]; + elevated_emis_output[isp] = elevated_emis_output_[isp + forcings_[i].offset]; } scream::mam_coupling::advance_tracer_data( - VertEmissionsDataReader_[i], *VertEmissionsHorizInterp_[i], ts, - vert_emiss_time_state_, vert_emis_data_[i], dry_atm_.p_mid, - dry_atm_.z_iface, vert_emis_output); + ElevatedEmissionsDataReader_[i], *ElevatedEmissionsHorizInterp_[i], ts, + elevated_emiss_time_state_, elevated_emis_data_[i], dry_atm_.p_mid, + dry_atm_.z_iface, elevated_emis_output); i++; Kokkos::fence(); } @@ -706,7 +706,7 @@ void MAMMicrophysics::run_impl(const double dt) { const auto zenith_angle = acos_cosine_zenith_; constexpr int gas_pcnst = mam_coupling::gas_pcnst(); - const auto& vert_emis_output = vert_emis_output_; + const auto& elevated_emis_output = elevated_emis_output_; const auto& extfrc = extfrc_; const auto& forcings = forcings_; constexpr int extcnt = mam4::gas_chemistry::extcnt; @@ -758,7 +758,7 @@ void MAMMicrophysics::run_impl(const double dt) { // We may need to move this line where we read files. forcings_in[i].file_alt_data = file_alt_data; for(int isec = 0; isec < forcings[i].nsectors; ++isec) { - const auto field = vert_emis_output[isec + forcings[i].offset]; + const auto field = elevated_emis_output[isec + forcings[i].offset]; forcings_in[i].fields_data[isec] = ekat::subview(field, icol); } } // extcnt for loop diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp index 5ecd8c62461..6be8f3e7fb5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -225,14 +225,14 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { // Vertical emission uses 9 files, here I am using std::vector to stote // instance of each file. - mam_coupling::TracerTimeState vert_emiss_time_state_; - std::vector> VertEmissionsDataReader_; - std::vector> VertEmissionsHorizInterp_; + mam_coupling::TracerTimeState elevated_emiss_time_state_; + std::vector> ElevatedEmissionsDataReader_; + std::vector> ElevatedEmissionsHorizInterp_; std::vector extfrc_lst_; - std::vector vert_emis_data_; - std::map vert_emis_file_name_; - std::map> vert_emis_var_names_; - view_2d vert_emis_output_[mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS]; + std::vector elevated_emis_data_; + std::map elevated_emis_file_name_; + std::map> elevated_emis_var_names_; + view_2d elevated_emis_output_[mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS]; view_3d extfrc_; mam_coupling::ForcingHelper forcings_[mam4::gas_chemistry::extcnt]; From a596e6bdc0f019c29d0712f8184f1a0b5efe7ad2 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Nov 2024 12:04:09 -0700 Subject: [PATCH 286/366] EAMxx - Rename tag names: 'elevated' replaces 'verti' --- .../cime_config/namelist_defaults_scream.xml | 40 +++++++++---------- ...mxx_mam_microphysics_process_interface.cpp | 4 +- .../mam/aero_microphys/input.yaml | 20 +++++----- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index a1435b03de1..9506cbdf3ab 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -293,29 +293,29 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/photolysis/RSF_GT200nm_v3.0_c080811.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc - - 20100101 + + 20100101 - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 39734fadf89..c53052b57ae 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -258,7 +258,7 @@ void MAMMicrophysics::set_grids( "num_a1", "num_a2", "num_a4", "soag"}; for(const auto &var_name : extfrc_lst_) { - std::string item_name = "mam4_" + var_name + "_verti_emiss_file_name"; + std::string item_name = "mam4_" + var_name + "_elevated_emiss_file_name"; const auto file_name = m_params.get(item_name); elevated_emis_file_name_[var_name] = file_name; } @@ -278,7 +278,7 @@ void MAMMicrophysics::set_grids( "num_a1_POM_ELEV_BB"}; elevated_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; - int elevated_emiss_cyclical_ymd = m_params.get("verti_emiss_ymd"); + int elevated_emiss_cyclical_ymd = m_params.get("elevated_emiss_ymd"); for(const auto &var_name : extfrc_lst_) { const auto file_name = elevated_emis_file_name_[var_name]; diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml index fa4538bb75a..d4d29091df4 100644 --- a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml +++ b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml @@ -27,16 +27,16 @@ atmosphere_processes: mam4_chlorine_loading_ymd : 20100101 mam4_rsf_file : ${SCREAM_DATA_DIR}/mam4xx/photolysis/RSF_GT200nm_v3.0_c080811.nc mam4_xs_long_file : ${SCREAM_DATA_DIR}/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc - verti_emiss_ymd : 20100101 - mam4_so2_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so2_elev_ne2np4_2010_clim_c20240726.nc - mam4_so4_a1_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_so4_a1_elev_ne2np4_2010_clim_c20240823.nc - mam4_so4_a2_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_so4_a2_elev_ne2np4_2010_clim_c20240823.nc - mam4_pom_a4_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_pom_a4_elev_ne2np4_2010_clim_c20240823.nc - mam4_bc_a4_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_bc_a4_elev_ne2np4_2010_clim_c20240823.nc - mam4_num_a1_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_num_a1_elev_ne2np4_2010_clim_c20240823.nc - mam4_num_a2_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_num_a2_elev_ne2np4_2010_clim_c20240823.nc - mam4_num_a4_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_num_a4_elev_ne2np4_2010_clim_c20240823.nc - mam4_soag_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_soag_elev_ne2np4_2010_clim_c20240823.nc + elevated_emiss_ymd : 20100101 + mam4_so2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so2_elev_ne2np4_2010_clim_c20240726.nc + mam4_so4_a1_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so4_a1_elev_ne2np4_2010_clim_c20240823.nc + mam4_so4_a2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so4_a2_elev_ne2np4_2010_clim_c20240823.nc + mam4_pom_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_pom_a4_elev_ne2np4_2010_clim_c20240823.nc + mam4_bc_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_bc_a4_elev_ne2np4_2010_clim_c20240823.nc + mam4_num_a1_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a1_elev_ne2np4_2010_clim_c20240823.nc + mam4_num_a2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a2_elev_ne2np4_2010_clim_c20240823.nc + mam4_num_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a4_elev_ne2np4_2010_clim_c20240823.nc + mam4_soag_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_soag_elev_ne2np4_2010_clim_c20240823.nc grids_manager: Type: Mesh Free From 589742d2e9ec5a28763c93c1c84aa450e0fa7db2 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Nov 2024 12:23:12 -0700 Subject: [PATCH 287/366] EAMxx - Replace 'verti' with 'elevated' in tag names; add missing lines. --- .../remap_emiss_ne4_ne30/shell_commands | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands index 606ca463f26..b2d0286b870 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands @@ -8,7 +8,7 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ -# Add aerosol microphysics process, force ne4pg2 +# Add aerosol microphysics process, force ne4pg2 # emission files and provide a ne4pg2->ne30pg2 mapping # file #------------------------------------------------------ @@ -16,15 +16,13 @@ alias ATMCHANGE='$CIMEROOT/../components/eamxx/scripts/atmchange' ATMCHANGE physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b -ATMCHANGE mam4_aero_microphys::mam4_so2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -ATMCHANGE mam4_aero_microphys::mam4_so4_a1_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -ATMCHANGE mam4_aero_microphys::mam4_so4_a2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -ATMCHANGE mam4_aero_microphys::mam4_pom_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -ATMCHANGE mam4_aero_microphys::mam4_bc_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -ATMCHANGE mam4_aero_microphys::mam4_num_a1_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -ATMCHANGE mam4_aero_microphys::mam4_num_a2_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -ATMCHANGE mam4_aero_microphys::mam4_num_a4_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b -ATMCHANGE mam4_aero_microphys::mam4_soag_verti_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_so2_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_so4_a1_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_so4_a2_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_pom_a4_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_bc_a4_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a1_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a2_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a4_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_soag_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b ATMCHANGE mam4_aero_microphys::aero_microphys_remap_file='${DIN_LOC_ROOT}/atm/scream/maps/map_ne4pg2_to_ne30pg2_nco_c20241108.nc' -b - - From c43116be9c3863eb9bd6357dfb191b56c0581885 Mon Sep 17 00:00:00 2001 From: AaronDonahue Date: Mon, 11 Nov 2024 16:00:09 -0700 Subject: [PATCH 288/366] This commit fixes a bug in defining an int layout for hyai and hybi --- components/eamxx/src/share/grid/mesh_free_grids_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index 446fba5229f..b083dfcfa48 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -147,7 +147,7 @@ add_geo_data (const nonconstgrid_ptr_type& grid) const if (geo_data_source=="CREATE_EMPTY_DATA") { using namespace ShortFieldTagsNames; FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); - FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()}); + FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()+1}); const auto units = ekat::units::Units::nondimensional(); auto lat = grid->create_geometry_data("lat" , grid->get_2d_scalar_layout(), units); @@ -242,7 +242,7 @@ load_vertical_coordinates (const nonconstgrid_ptr_type& grid, const std::string& using namespace ekat::units; FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); - FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()}); + FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()+1}); Units nondim = Units::nondimensional(); Units mbar (100*Pa,"mb"); From a1705ddd0710a5b17aee43904c931ab05185e560 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 12 Nov 2024 10:08:32 -0700 Subject: [PATCH 289/366] Update .mergify.yml List all checks individually --- .mergify.yml | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 6c9318ce5b6..ce6a74d904a 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -7,14 +7,38 @@ merge_protections: - "#approved-reviews-by >= 1" # At least 1 approval - "#changes-requested-reviews-by == 0" # No reviewer asked for changes - or: - - check-success="gcc-openmp / .*" - - check-skipped="gcc-openmp / .*" + - check-success="gcc-openmp / dbg" + - check-skipped="gcc-openmp / dbg" - or: - - check-success="gcc-cuda / .*" - - check-skipped="gcc-cuda / .*" + - check-success="gcc-openmp / sp" + - check-skipped="gcc-openmp / sp" - or: - - check-success="cpu-gcc / .*" - - check-skipped="cpu-gcc / .*" + - check-success="gcc-openmp / fpe" + - check-skipped="gcc-openmp / fpe" + - or: + - check-success="gcc-openmp / opt" + - check-skipped="gcc-openmp / opt" + - or: + - check-success="gcc-cuda / dbg" + - check-skipped="gcc-cuda / dbg" + - or: + - check-success="gcc-cuda / sp" + - check-skipped="gcc-cuda / sp" + - or: + - check-success="gcc-cuda / opt" + - check-skipped="gcc-cuda / opt" + - or: + - check-success="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" + - check-skipped="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" + - or: + - check-success="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" + - check-skipped="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" + - or: + - check-success="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" + - check-skipped="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" + - or: + - check-success="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" + - check-skipped="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" - or: - check-success=cpu-gcc - check-skipped=cpu-gcc From 0939ad35b71e8976b047068e0e4d3a91b9f99051 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 12 Nov 2024 10:32:06 -0700 Subject: [PATCH 290/366] Mergify: enable pull request automerge --- .mergify.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index ce6a74d904a..e46f2208ed5 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -42,3 +42,13 @@ merge_protections: - or: - check-success=cpu-gcc - check-skipped=cpu-gcc + +pull_request_rules: + - name: Automatic merge when CI passes and approved + conditions: + - check-success="Mergify Merge Protections" # Pass if merge protection rule passes + - "label=CI: automerge" + - base=master + actions: + merge: + method: merge From d4b7f47bb6386213c0603ecb6a39b22ff2db1d24 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Tue, 12 Nov 2024 16:06:07 -0500 Subject: [PATCH 291/366] Update machine and compiler files for Frontier - Recover ADIOS support - Load libfabric/1.15.2.0 module - Fix typos in configuration files --- .../machines/cmake_macros/amdclanggpu_frontier.cmake | 2 -- .../machines/cmake_macros/crayclanggpu_frontier.cmake | 2 -- .../machines/cmake_macros/gnugpu_frontier.cmake | 1 - cime_config/machines/config_machines.xml | 11 ++++------- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake index 4412ea0de7b..1deebdac85d 100644 --- a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake @@ -13,8 +13,6 @@ string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") diff --git a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake index a37ccde439e..92567416c56 100644 --- a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake @@ -52,8 +52,6 @@ endif() # https://github.com/E3SM-Project/E3SM/pull/5208 string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") diff --git a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake index 6ca4b83d9c2..7a29a5ca154 100644 --- a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake @@ -14,7 +14,6 @@ string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index fa1d767fbb5..8d210436c43 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1069,7 +1069,7 @@ /usr/share/lmod/lmod/libexec/lmod python - Core/24.07 + Core Core/24.07 PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/15.0.1 @@ -1082,7 +1082,7 @@ - Core/24.07 + Core Core/24.07 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.4.0 @@ -1091,7 +1091,7 @@ - Core/24.07 + Core Core/24.07 PrgEnv-cray PrgEnv-gnu/8.3.3 gcc gcc/12.2.0 @@ -1106,6 +1106,7 @@ subversion git zlib + libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 @@ -1139,9 +1140,6 @@ spread threads - From 035d8c2ae5a453e64ae6f20931a33aa71c39d9b3 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 12 Nov 2024 13:54:03 -0700 Subject: [PATCH 292/366] Mergify: set commit message for automerge --- .mergify.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index e46f2208ed5..314459ef1c8 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -52,3 +52,10 @@ pull_request_rules: actions: merge: method: merge + commit_message_template: | + Merge pull request #{{number}} from {{head}} + + Automatically merged using mergify + PR title: {{title}} + PR author: {{author}} + PR labels: {{label}} From b4e18d2d7fc08fd32ef1489b54726328f610f6aa Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 12 Nov 2024 15:35:32 -0700 Subject: [PATCH 293/366] Mergify: remove redundant merge protection condition in automerge rule Mergify already includes merge protection rules inside the pull request rules --- .mergify.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index 314459ef1c8..ee3e186f509 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -46,7 +46,6 @@ merge_protections: pull_request_rules: - name: Automatic merge when CI passes and approved conditions: - - check-success="Mergify Merge Protections" # Pass if merge protection rule passes - "label=CI: automerge" - base=master actions: From be3127b6e6e15ae17f0b9477788c84957a386cc4 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Tue, 12 Nov 2024 16:39:19 -0600 Subject: [PATCH 294/366] Set default config_check_ssh_consistency for oQU240wLI to false --- .../mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 3b93400f4e8..f1ab05d2207 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -630,6 +630,7 @@ .false. .true. +.false. .false. .false. .false. From 2bb921e202de32c9801f389d3546bd4cbfb55e62 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 13 Nov 2024 10:32:04 -0700 Subject: [PATCH 295/366] Quick fix for baseline generation The seperate small kernel testing does not need to be (and should not) run when generating baselines. --- components/eamxx/src/physics/p3/CMakeLists.txt | 2 ++ components/eamxx/src/physics/p3/tests/CMakeLists.txt | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index 2a630438049..58a026e1601 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -69,6 +69,8 @@ if (SCREAM_P3_SMALL_KERNELS) add_library(p3 ${P3_SRCS} ${P3_SK_SRCS}) else() add_library(p3 ${P3_SRCS}) + # If small kernels are ON, we don't need a separate executable to test them. + # Also, we never want to generate baselines with this separate executable if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(p3_sk ${P3_SRCS} ${P3_SK_SRCS}) # Always build p3_sk with SCREAM_P3_SMALL_KERNELS on diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 129e3c455e2..d4fb90b1be3 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -74,7 +74,9 @@ if (SCREAM_ENABLE_BASELINE_TESTS) ${FORCE_RUN_DIFF_FAILS}) endif() -if (NOT SCREAM_P3_SMALL_KERNELS) +# If small kernels are ON, we don't need a separate executable to test them. +# Also, we never want to generate baselines with this separate executable +if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" LIBS p3_sk p3_test_infra EXE_ARGS "--args ${BASELINE_FILE_ARG}" From 5dc66fc292bb39d9317fe49710f8691747ff2f8d Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 13 Nov 2024 12:55:38 -0700 Subject: [PATCH 296/366] Fixup small kernel situation --- .../p3/disp/p3_check_values_impl_disp.cpp | 3 +-- .../physics/p3/disp/p3_cloud_sed_impl_disp.cpp | 3 +-- .../physics/p3/disp/p3_rain_sed_impl_disp.cpp | 10 +++++----- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 17 +++++------------ 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp index cc13e99bc91..5bac8306477 100644 --- a/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp @@ -16,7 +16,7 @@ ::check_values_disp(const uview_2d& qv, const uview_2d using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); - + Kokkos::parallel_for( "p3_check_values", policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -32,4 +32,3 @@ ::check_values_disp(const uview_2d& qv, const uview_2d } // namespace p3 } // namespace scream - diff --git a/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp index 8b755be4857..4b528663d23 100644 --- a/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp @@ -48,7 +48,7 @@ ::cloud_sedimentation_disp( } cloud_sedimentation( - ekat::subview(qc_incld, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(cld_frac_l, i), + ekat::subview(qc_incld, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(cld_frac_l, i), ekat::subview(acn, i), ekat::subview(inv_dz, i), dnu, team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, do_predict_nc, ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(nc_incld, i), ekat::subview(mu_c, i), ekat::subview(lamc, i), ekat::subview(qc_tend, i), @@ -60,4 +60,3 @@ ::cloud_sedimentation_disp( } // namespace p3 } // namespace scream - diff --git a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp index d152313dc7f..a1b38afe8ed 100644 --- a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp @@ -45,11 +45,11 @@ ::rain_sedimentation_disp( // Rain sedimentation: (adaptive substepping) rain_sedimentation( - ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofacr, i), ekat::subview(cld_frac_r, i), - ekat::subview(inv_dz, i), ekat::subview(qr_incld, i), - team, workspace, vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, - ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(nr_incld, i), ekat::subview(mu_r, i), - ekat::subview(lamr, i), ekat::subview(precip_liq_flux, i), + ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofacr, i), ekat::subview(cld_frac_r, i), + ekat::subview(inv_dz, i), ekat::subview(qr_incld, i), + team, workspace, vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, + ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(nr_incld, i), ekat::subview(mu_r, i), + ekat::subview(lamr, i), ekat::subview(precip_liq_flux, i), ekat::subview(qr_tend, i), ekat::subview(nr_tend, i), precip_liq_surf(i), runtime_options); }); diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index d4fb90b1be3..d45314ccb9c 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -77,22 +77,15 @@ endif() # If small kernels are ON, we don't need a separate executable to test them. # Also, we never want to generate baselines with this separate executable if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_ONLY_GENERATE_BASELINES) - CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" + # Note: Only the p3_main test does something different when + # small kernels are on. The SK dispatch routines are mostly trivial + # and it's not worth adding tons of test infrastructure to support + # BFB unit tests for these. + CreateUnitTest(p3_sk_tests "p3_main_unit_tests.cpp" LIBS p3_sk p3_test_infra EXE_ARGS "--args ${BASELINE_FILE_ARG}" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3_sk;physics") - - # Make sure that a diff in the two implementation triggers a failed test (in debug only) - # No need to run lots of different thread counts. - if (SCREAM_ENABLE_BASELINE_TESTS) - CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3_sk p3_test_infra - EXE_ARGS "--args ${BASELINE_FILE_ARG}" - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - LABELS "p3_sk;physics;fail" - ${FORCE_RUN_DIFF_FAILS}) - endif() endif() CreateUnitTest(p3_run_and_cmp "p3_run_and_cmp.cpp" From 6318f41520e74a266ffe89ea60342372c8eb7f31 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 13 Nov 2024 12:26:02 -0700 Subject: [PATCH 297/366] Mergify: fix merge proteciton rule The name of skipped jobs from a matrix needed to be fixed --- .mergify.yml | 51 +++++++++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index ee3e186f509..315cad81cc6 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -7,38 +7,25 @@ merge_protections: - "#approved-reviews-by >= 1" # At least 1 approval - "#changes-requested-reviews-by == 0" # No reviewer asked for changes - or: - - check-success="gcc-openmp / dbg" - - check-skipped="gcc-openmp / dbg" - - or: - - check-success="gcc-openmp / sp" - - check-skipped="gcc-openmp / sp" - - or: - - check-success="gcc-openmp / fpe" - - check-skipped="gcc-openmp / fpe" - - or: - - check-success="gcc-openmp / opt" - - check-skipped="gcc-openmp / opt" - - or: - - check-success="gcc-cuda / dbg" - - check-skipped="gcc-cuda / dbg" - - or: - - check-success="gcc-cuda / sp" - - check-skipped="gcc-cuda / sp" - - or: - - check-success="gcc-cuda / opt" - - check-skipped="gcc-cuda / opt" - - or: - - check-success="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" - - check-skipped="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" - - or: - - check-success="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" - - check-skipped="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" - - or: - - check-success="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" - - check-skipped="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" - - or: - - check-success="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" - - check-skipped="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" + - and: + - check-success="gcc-openmp / dbg" + - check-success="gcc-openmp / sp" + - check-success="gcc-openmp / fpe" + - check-success="gcc-openmp / opt" + - check-skipped~="gcc-openmp / .*" + - or: + - and: + - check-success="gcc-cuda / dbg" + - check-success="gcc-cuda / sp" + - check-success="gcc-cuda / opt" + - check-skipped~="gcc-cuda / .*" + - or: + - and: + - check-success="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" + - check-success="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" + - check-success="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" + - check-success="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" + - check-skipped~="cpu-gcc / .*" - or: - check-success=cpu-gcc - check-skipped=cpu-gcc From dfbfcfd324aee44ddd39a27b76b2c8848fe64a8c Mon Sep 17 00:00:00 2001 From: Peter Andrew Bogenschutz Date: Wed, 13 Nov 2024 13:34:33 -0800 Subject: [PATCH 298/366] make latitude absolute value in the relative checking to make it compatible for southern hemisphere cases --- components/eamxx/src/share/iop/intensive_observation_period.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 32b120c5700..1d5f7c7ba33 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -270,7 +270,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),(Real)0.1); + std::max(std::fabs(m_params.get("target_latitude")),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), From db5b35dbb7ccf9dda2f7c7759870945938f73aa0 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 13 Nov 2024 12:26:02 -0700 Subject: [PATCH 299/366] Mergify: fix merge proteciton rule The name of skipped jobs from a matrix needed to be fixed --- .mergify.yml | 51 +++++++++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index ee3e186f509..9c2b8fe13f1 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -7,38 +7,25 @@ merge_protections: - "#approved-reviews-by >= 1" # At least 1 approval - "#changes-requested-reviews-by == 0" # No reviewer asked for changes - or: - - check-success="gcc-openmp / dbg" - - check-skipped="gcc-openmp / dbg" - - or: - - check-success="gcc-openmp / sp" - - check-skipped="gcc-openmp / sp" - - or: - - check-success="gcc-openmp / fpe" - - check-skipped="gcc-openmp / fpe" - - or: - - check-success="gcc-openmp / opt" - - check-skipped="gcc-openmp / opt" - - or: - - check-success="gcc-cuda / dbg" - - check-skipped="gcc-cuda / dbg" - - or: - - check-success="gcc-cuda / sp" - - check-skipped="gcc-cuda / sp" - - or: - - check-success="gcc-cuda / opt" - - check-skipped="gcc-cuda / opt" - - or: - - check-success="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" - - check-skipped="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" - - or: - - check-success="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" - - check-skipped="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" - - or: - - check-success="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" - - check-skipped="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" - - or: - - check-success="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" - - check-skipped="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" + - and: + - check-success="gcc-openmp / dbg" + - check-success="gcc-openmp / sp" + - check-success="gcc-openmp / fpe" + - check-success="gcc-openmp / opt" + - check-skipped={% raw %}gcc-openmp / ${{ matrix.build_type }}{% endraw %} + - or: + - and: + - check-success="gcc-cuda / dbg" + - check-success="gcc-cuda / sp" + - check-success="gcc-cuda / opt" + - check-skipped={% raw %}gcc-cuda / ${{ matrix.build_type }}{% endraw %} + - or: + - and: + - check-success="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" + - check-success="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" + - check-success="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" + - check-success="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" + - check-skipped={% raw %}cpu-gcc / ${{ matrix.test.short_name }}{% endraw %} - or: - check-success=cpu-gcc - check-skipped=cpu-gcc From b7211bd09f8b368a86227dbb390b21f0078854cf Mon Sep 17 00:00:00 2001 From: Peter Andrew Bogenschutz Date: Wed, 13 Nov 2024 15:56:01 -0800 Subject: [PATCH 300/366] add option to prescribe an invariant solar constant --- .../eamxx/cime_config/namelist_defaults_scream.xml | 5 +++++ .../physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 10 ++++++++++ .../physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 9506cbdf3ab..4ab040a194b 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -495,6 +495,11 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 -9999.0 0.0 + + + -9999.0 + 551.58 + 1 2 4 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 758baa80b7d..1634473b9f8 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -620,6 +620,10 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { m_orbital_obliq = m_params.get("orbital_obliquity" ,-9999); m_orbital_mvelp = m_params.get("orbital_mvelp" ,-9999); + // Value for prescribing an invariant solar constant (i.e. total solar irradiance at + // TOA). Used for idealized experiments such as RCE. Disabled when value is less than 0. + m_fixed_total_solar_irradiance = m_params.get("fixed_total_solar_irradiance", -9999); + // Determine whether or not we are using a fixed solar zenith angle (positive value) m_fixed_solar_zenith_angle = m_params.get("Fixed Solar Zenith Angle", -9999); @@ -823,6 +827,7 @@ void RRTMGPRadiation::run_impl (const double dt) { auto eccen = m_orbital_eccen; auto obliq = m_orbital_obliq; auto mvelp = m_orbital_mvelp; + auto fixed_total_solar_irradiance = m_fixed_total_solar_irradiance; if (eccen >= 0 && obliq >= 0 && mvelp >= 0) { // use fixed orbital parameters; to force this, we need to set // orbital_year to SHR_ORB_UNDEF_INT, which is exposed through @@ -840,6 +845,11 @@ void RRTMGPRadiation::run_impl (const double dt) { shr_orb_decl_c2f(calday, eccen, mvelpp, lambm0, obliqr, &delta, &eccf); + // Overwrite eccf if using a fixed solar constant. + if (fixed_total_solar_irradiance >= 0){ + eccf = fixed_total_solar_irradiance/1360.9; + } + // Precompute VMR for all gases, on all cols, before starting the chunks loop // // h2o is taken from qv diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 329c7eeaba8..02a047cce59 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -95,6 +95,11 @@ class RRTMGPRadiation : public AtmosphereProcess { Real m_orbital_obliq; // Obliquity Real m_orbital_mvelp; // Vernal Equinox Mean Longitude of Perihelion + // Value for prescribing an invariant solar constant (i.e. total solar irradiance + // at TOA). Used for idealized experiments such as RCE. This is only used when a + // positive value is supplied. + Real m_fixed_total_solar_irradiance; + // Fixed solar zenith angle to use for shortwave calculations // This is only used if a positive value is supplied Real m_fixed_solar_zenith_angle; From 50815e27dd000745d273ef8da91fdfb9d3cbdd04 Mon Sep 17 00:00:00 2001 From: Peter Andrew Bogenschutz Date: Wed, 13 Nov 2024 16:01:02 -0800 Subject: [PATCH 301/366] move placement of line where fixed solar irradiance is defined --- .../eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1634473b9f8..0709bb1a37f 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -827,7 +827,6 @@ void RRTMGPRadiation::run_impl (const double dt) { auto eccen = m_orbital_eccen; auto obliq = m_orbital_obliq; auto mvelp = m_orbital_mvelp; - auto fixed_total_solar_irradiance = m_fixed_total_solar_irradiance; if (eccen >= 0 && obliq >= 0 && mvelp >= 0) { // use fixed orbital parameters; to force this, we need to set // orbital_year to SHR_ORB_UNDEF_INT, which is exposed through @@ -846,6 +845,7 @@ void RRTMGPRadiation::run_impl (const double dt) { obliqr, &delta, &eccf); // Overwrite eccf if using a fixed solar constant. + auto fixed_total_solar_irradiance = m_fixed_total_solar_irradiance; if (fixed_total_solar_irradiance >= 0){ eccf = fixed_total_solar_irradiance/1360.9; } From 8e72936483027815eddc124c97316e4191232192 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Nov 2024 20:10:50 -0700 Subject: [PATCH 302/366] EAMxx: Interface for reading file and setting index_season_lai --- ...mxx_mam_microphysics_process_interface.cpp | 9 +++ ...mxx_mam_microphysics_process_interface.hpp | 4 ++ .../mam/readfiles/find_season_index_utils.hpp | 62 +++++++++++++++++++ externals/mam4xx | 2 +- 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index c53052b57ae..a284178a8b3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -3,6 +3,7 @@ // impl namespace for some driver level functions for microphysics #include "readfiles/photo_table_utils.cpp" +#include "readfiles/find_season_index_utils.hpp" #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" namespace scream { @@ -332,6 +333,14 @@ void MAMMicrophysics::set_grids( "MAX_NUM_ELEVATED_EMISSIONS_FIELDS in tracer_reader_utils.hpp \n"); } // Tracer external forcing data + + { + std::string season_wes_file ="season_wes.nc"; + const auto& clat = col_latitudes_; + mam_coupling::find_season_index_reader(season_wes_file, + clat, + index_season_lai_); + } } // set_grids // ================================================================ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp index 6be8f3e7fb5..d555fe82024 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -25,6 +25,8 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { using view_1d_host = typename KT::view_1d::HostMirror; + using view_int_2d = typename KT::template view_2d; + // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; @@ -239,6 +241,8 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { view_1d_host acos_cosine_zenith_host_; view_1d acos_cosine_zenith_; + view_int_2d index_season_lai_; + }; // MAMMicrophysics } // namespace scream diff --git a/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp new file mode 100644 index 00000000000..70f743b2376 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp @@ -0,0 +1,62 @@ +#ifndef EAMXX_MAM_FIND_SEASON_INDEX_UTILS +#define EAMXX_MAM_FIND_SEASON_INDEX_UTILS + +#include +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include + +namespace scream::mam_coupling { + +using ExeSpace = typename KT::ExeSpace; +using ESU = ekat::ExeSpaceUtils; + +// views for single- and multi-column data +using view_1d = typename KT::template view_1d; +using const_view_1d = typename KT::template view_1d; + +using view_int_1d = typename KT::template view_1d; +using view_int_2d = typename KT::template view_2d; +using view_int_3d = typename KT::template view_3d; + +using view_1d_host = typename KT::view_1d::HostMirror; +using view_3d_host = typename KT::view_3d::HostMirror; +using view_int_3d_host = typename KT::view_3d::HostMirror; + + +inline void find_season_index_reader(const std::string& season_wes_file, + const const_view_1d& clat, + view_int_2d &index_season_lai) +{ + const int plon= clat.extent(0); + scorpio::register_file(season_wes_file, scorpio::Read); + + const int nlat_lai = scorpio::get_dimlen(season_wes_file, "lat"); + const int npft_lai = scorpio::get_dimlen(season_wes_file, "pft"); + view_1d_host lat_lai_host("lat_lai_host", nlat_lai); + view_int_3d_host wk_lai_host("wk_lai_host", nlat_lai, npft_lai, 12); + + scorpio::read_var(season_wes_file, "lat", lat_lai_host.data()); + scorpio::read_var(season_wes_file, "season_wes", wk_lai_host.data()); + scorpio::release_file(season_wes_file); + + view_int_3d wk_lai("wk_lai", nlat_lai, npft_lai, 12); + + view_1d lat_lai("lat_lai", nlat_lai); + Kokkos::deep_copy(lat_lai, lat_lai_host); + Kokkos::deep_copy(wk_lai, wk_lai_host); + + // output + index_season_lai=view_int_2d("index_season_lai", plon,12); + const auto policy = ESU::get_default_team_policy(plon, 1); + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const Team &team) { + const int j = team.league_rank(); + const auto index_season_lai_at_j = ekat::subview(index_season_lai, j); + mam4::mo_drydep::find_season_index(clat(j), lat_lai, nlat_lai, wk_lai, + index_season_lai_at_j); + }); +} + +} // namespace scream::mam_coupling +#endif // diff --git a/externals/mam4xx b/externals/mam4xx index aae46807bf5..5e21c26f066 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit aae46807bf58d6ffcbc6db620c27568450b2d040 +Subproject commit 5e21c26f0662641cfd6522d5e96315f67cd7a4b4 From 79cb356562f29c049175ffb51555bf9cd348a9b3 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 13 Nov 2024 09:45:12 -0700 Subject: [PATCH 303/366] Use host version for find_season_index and add function description --- .../mam/readfiles/find_season_index_utils.hpp | 67 ++++++++++++------- externals/mam4xx | 2 +- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp index 70f743b2376..c7778b74421 100644 --- a/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp @@ -12,17 +12,22 @@ using ExeSpace = typename KT::ExeSpace; using ESU = ekat::ExeSpaceUtils; // views for single- and multi-column data -using view_1d = typename KT::template view_1d; -using const_view_1d = typename KT::template view_1d; -using view_int_1d = typename KT::template view_1d; +using const_view_1d = typename KT::template view_1d; using view_int_2d = typename KT::template view_2d; -using view_int_3d = typename KT::template view_3d; using view_1d_host = typename KT::view_1d::HostMirror; using view_3d_host = typename KT::view_3d::HostMirror; using view_int_3d_host = typename KT::view_3d::HostMirror; +using view_int_2d_host = typename KT::view_2d::HostMirror; +/** + * @brief Reads the season index from the given file and computes the season indices based on latitudes. + * + * @param[in] season_wes_file The path to the season_wes.nc file. + * @param[in] clat A 1D view of latitude values in degrees. + * @param[out] index_season_lai A 2D view to store the computed season indices. Note that indices are in C++ (starting from zero). + */ inline void find_season_index_reader(const std::string& season_wes_file, const const_view_1d& clat, @@ -33,30 +38,42 @@ inline void find_season_index_reader(const std::string& season_wes_file, const int nlat_lai = scorpio::get_dimlen(season_wes_file, "lat"); const int npft_lai = scorpio::get_dimlen(season_wes_file, "pft"); - view_1d_host lat_lai_host("lat_lai_host", nlat_lai); - view_int_3d_host wk_lai_host("wk_lai_host", nlat_lai, npft_lai, 12); - scorpio::read_var(season_wes_file, "lat", lat_lai_host.data()); - scorpio::read_var(season_wes_file, "season_wes", wk_lai_host.data()); - scorpio::release_file(season_wes_file); + view_1d_host lat_lai("lat_lai", nlat_lai); + view_int_2d_host wk_lai_temp("wk_lai", npft_lai, nlat_lai); + view_int_3d_host wk_lai("wk_lai", nlat_lai, npft_lai, 12); - view_int_3d wk_lai("wk_lai", nlat_lai, npft_lai, 12); - - view_1d lat_lai("lat_lai", nlat_lai); - Kokkos::deep_copy(lat_lai, lat_lai_host); - Kokkos::deep_copy(wk_lai, wk_lai_host); - - // output - index_season_lai=view_int_2d("index_season_lai", plon,12); - const auto policy = ESU::get_default_team_policy(plon, 1); - Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const Team &team) { - const int j = team.league_rank(); - const auto index_season_lai_at_j = ekat::subview(index_season_lai, j); - mam4::mo_drydep::find_season_index(clat(j), lat_lai, nlat_lai, wk_lai, - index_season_lai_at_j); + scorpio::read_var(season_wes_file, "lat", lat_lai.data()); + + Kokkos::MDRangePolicy> + policy_wk_lai({0, 0}, {nlat_lai, npft_lai}); + + // loop over time to get all 12 instantence of season_wes + for (int itime = 0; itime < 12; ++itime) + { + scorpio::read_var(season_wes_file, "season_wes", wk_lai_temp.data(),itime); + // copy data from wk_lai_temp to wk_lai. + // NOTE: season_wes has different layout that wk_lai + Kokkos::parallel_for("copy_to_wk_lai", policy_wk_lai, + [&](const int j, const int k) { + wk_lai(j, k, itime) = wk_lai_temp(k, j); }); -} + Kokkos::fence(); + } + scorpio::release_file(season_wes_file); + + index_season_lai =view_int_2d("index_season_lai", plon, 12); + const view_int_2d_host index_season_lai_host= Kokkos::create_mirror_view(index_season_lai); + + const view_1d_host clat_host= Kokkos::create_mirror_view(clat); + Kokkos::deep_copy(clat_host, clat); + // Computation is performed on the host + mam4::mo_drydep::find_season_index(clat_host, lat_lai, nlat_lai, wk_lai, + index_season_lai_host); + Kokkos::deep_copy(index_season_lai, index_season_lai_host); + + +} } // namespace scream::mam_coupling #endif // diff --git a/externals/mam4xx b/externals/mam4xx index 5e21c26f066..d6cb4c004a6 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 5e21c26f0662641cfd6522d5e96315f67cd7a4b4 +Subproject commit d6cb4c004a686664dc9d5885837f69c46be2f611 From a97a7730dad27dde08fb313440c368afa8c298e1 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 13 Nov 2024 11:39:41 -0700 Subject: [PATCH 304/366] EAMxx: Obtain season_wes_file from input file and use MAM4xx main branch --- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 ++ .../physics/mam/eamxx_mam_microphysics_process_interface.cpp | 2 +- .../eamxx/tests/single-process/mam/aero_microphys/input.yaml | 2 +- externals/mam4xx | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 9506cbdf3ab..195f46847ef 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -295,6 +295,8 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc 20100101 + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/drydep/season_wes.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne30pg2_c20241008.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index a284178a8b3..57eaf927548 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -335,7 +335,7 @@ void MAMMicrophysics::set_grids( } // Tracer external forcing data { - std::string season_wes_file ="season_wes.nc"; + const std::string season_wes_file = m_params.get("mam4_season_wes_file"); const auto& clat = col_latitudes_; mam_coupling::find_season_index_reader(season_wes_file, clat, diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml index d4d29091df4..9348a04a21d 100644 --- a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml +++ b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml @@ -37,7 +37,7 @@ atmosphere_processes: mam4_num_a2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a2_elev_ne2np4_2010_clim_c20240823.nc mam4_num_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a4_elev_ne2np4_2010_clim_c20240823.nc mam4_soag_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_soag_elev_ne2np4_2010_clim_c20240823.nc - + mam4_season_wes_file : ${SCREAM_DATA_DIR}/mam4xx/drydep/season_wes.nc grids_manager: Type: Mesh Free geo_data_source: IC_FILE diff --git a/externals/mam4xx b/externals/mam4xx index d6cb4c004a6..4431bbd1eef 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit d6cb4c004a686664dc9d5885837f69c46be2f611 +Subproject commit 4431bbd1eef46de25be3a04e7091c9255bd0b819 From 1f9563a722ff4e72f1c8dd93012b7c13b4646f4b Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 13 Nov 2024 11:44:09 -0700 Subject: [PATCH 305/366] EAMxx: apply clang format to find_season_index_utils.hpp. --- .../mam/readfiles/find_season_index_utils.hpp | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp index c7778b74421..2550673bd9c 100644 --- a/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp @@ -2,9 +2,10 @@ #define EAMXX_MAM_FIND_SEASON_INDEX_UTILS #include +#include + #include "share/io/scorpio_input.hpp" #include "share/io/scream_scorpio_interface.hpp" -#include namespace scream::mam_coupling { @@ -13,27 +14,28 @@ using ESU = ekat::ExeSpaceUtils; // views for single- and multi-column data -using const_view_1d = typename KT::template view_1d; -using view_int_2d = typename KT::template view_2d; +using const_view_1d = typename KT::template view_1d; +using view_int_2d = typename KT::template view_2d; -using view_1d_host = typename KT::view_1d::HostMirror; -using view_3d_host = typename KT::view_3d::HostMirror; +using view_1d_host = typename KT::view_1d::HostMirror; +using view_3d_host = typename KT::view_3d::HostMirror; using view_int_3d_host = typename KT::view_3d::HostMirror; using view_int_2d_host = typename KT::view_2d::HostMirror; /** - * @brief Reads the season index from the given file and computes the season indices based on latitudes. + * @brief Reads the season index from the given file and computes the season + * indices based on latitudes. * * @param[in] season_wes_file The path to the season_wes.nc file. * @param[in] clat A 1D view of latitude values in degrees. - * @param[out] index_season_lai A 2D view to store the computed season indices. Note that indices are in C++ (starting from zero). + * @param[out] index_season_lai A 2D view to store the computed season indices. + * Note that indices are in C++ (starting from zero). */ -inline void find_season_index_reader(const std::string& season_wes_file, - const const_view_1d& clat, - view_int_2d &index_season_lai) -{ - const int plon= clat.extent(0); +inline void find_season_index_reader(const std::string &season_wes_file, + const const_view_1d &clat, + view_int_2d &index_season_lai) { + const int plon = clat.extent(0); scorpio::register_file(season_wes_file, scorpio::Read); const int nlat_lai = scorpio::get_dimlen(season_wes_file, "lat"); @@ -46,34 +48,32 @@ inline void find_season_index_reader(const std::string& season_wes_file, scorpio::read_var(season_wes_file, "lat", lat_lai.data()); Kokkos::MDRangePolicy> - policy_wk_lai({0, 0}, {nlat_lai, npft_lai}); + policy_wk_lai({0, 0}, {nlat_lai, npft_lai}); // loop over time to get all 12 instantence of season_wes - for (int itime = 0; itime < 12; ++itime) - { - scorpio::read_var(season_wes_file, "season_wes", wk_lai_temp.data(),itime); - // copy data from wk_lai_temp to wk_lai. - // NOTE: season_wes has different layout that wk_lai - Kokkos::parallel_for("copy_to_wk_lai", policy_wk_lai, - [&](const int j, const int k) { - wk_lai(j, k, itime) = wk_lai_temp(k, j); - }); - Kokkos::fence(); + for(int itime = 0; itime < 12; ++itime) { + scorpio::read_var(season_wes_file, "season_wes", wk_lai_temp.data(), itime); + // copy data from wk_lai_temp to wk_lai. + // NOTE: season_wes has different layout that wk_lai + Kokkos::parallel_for("copy_to_wk_lai", policy_wk_lai, + [&](const int j, const int k) { + wk_lai(j, k, itime) = wk_lai_temp(k, j); + }); + Kokkos::fence(); } scorpio::release_file(season_wes_file); - index_season_lai =view_int_2d("index_season_lai", plon, 12); - const view_int_2d_host index_season_lai_host= Kokkos::create_mirror_view(index_season_lai); + index_season_lai = view_int_2d("index_season_lai", plon, 12); + const view_int_2d_host index_season_lai_host = + Kokkos::create_mirror_view(index_season_lai); - const view_1d_host clat_host= Kokkos::create_mirror_view(clat); + const view_1d_host clat_host = Kokkos::create_mirror_view(clat); Kokkos::deep_copy(clat_host, clat); // Computation is performed on the host mam4::mo_drydep::find_season_index(clat_host, lat_lai, nlat_lai, wk_lai, - index_season_lai_host); + index_season_lai_host); Kokkos::deep_copy(index_season_lai, index_season_lai_host); - - } } // namespace scream::mam_coupling #endif // From 0d1fdc3448ca4d76b0c2fc344520de36a2cebd66 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 13 Nov 2024 13:21:52 -0700 Subject: [PATCH 306/366] EAMxx: Obtain season_wes.nc file. --- .../eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt index e1ba85a685f..6f6f8b364b8 100644 --- a/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt @@ -39,6 +39,7 @@ set (TEST_INPUT_FILES scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a2_elev_ne2np4_2010_clim_c20240823.nc scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a4_elev_ne2np4_2010_clim_c20240823.nc scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_soag_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/drydep/season_wes.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) From 9fbdbb39e864b46c108b35d068c729060e7840d9 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 13 Nov 2024 16:22:28 -0700 Subject: [PATCH 307/366] EAMxx: remove unused code. --- .../src/physics/mam/readfiles/find_season_index_utils.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp index 2550673bd9c..2e930cc65cb 100644 --- a/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp @@ -9,16 +9,12 @@ namespace scream::mam_coupling { -using ExeSpace = typename KT::ExeSpace; -using ESU = ekat::ExeSpaceUtils; - // views for single- and multi-column data using const_view_1d = typename KT::template view_1d; using view_int_2d = typename KT::template view_2d; using view_1d_host = typename KT::view_1d::HostMirror; -using view_3d_host = typename KT::view_3d::HostMirror; using view_int_3d_host = typename KT::view_3d::HostMirror; using view_int_2d_host = typename KT::view_2d::HostMirror; From a9850ad7484c9c4088cfd93838df109fc52a0750 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 14 Nov 2024 11:24:40 -0700 Subject: [PATCH 308/366] Fixes for p3 tests CMake settings p3_tests needs to have baseline_gen label. Also, avoid doing thread spreads when generating. --- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index d45314ccb9c..9d2b2419c97 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -50,18 +50,23 @@ endif() if (SCREAM_ENABLE_BASELINE_TESTS) if (SCREAM_ONLY_GENERATE_BASELINES) set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data") + # We don't want to do thread spreads when generating. That + # could cause race conditions in the file system. + set(P3_THREADS "${SCREAM_TEST_MAX_THREADS}") else() set(BASELINE_FILE_ARG "-c -b ${SCREAM_BASELINES_DIR}/data") + set(P3_THREADS "1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}") endif() else() set(BASELINE_FILE_ARG "-n") # no baselines + set(P3_THREADS "1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}") endif() CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" LIBS p3 p3_test_infra EXE_ARGS "--args ${BASELINE_FILE_ARG}" - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3;physics") + THREADS ${P3_THREADS} + LABELS "p3;physics;baseline_gen;baseline_cmp") # Make sure that a diff in the two implementation triggers a failed test (in debug only) # No need to run lots of different thread counts. @@ -85,15 +90,16 @@ if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_ONLY_GENERATE_BASELINES) LIBS p3_sk p3_test_infra EXE_ARGS "--args ${BASELINE_FILE_ARG}" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3_sk;physics") + LABELS "p3_sk;physics;baseline_cmp") endif() +# Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it CreateUnitTest(p3_run_and_cmp "p3_run_and_cmp.cpp" LIBS p3 p3_test_infra EXCLUDE_MAIN_CPP THREADS ${SCREAM_TEST_MAX_THREADS} EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "p3;physics") + LABELS "p3;physics;baseline_gen;baseline_cmp") # Make sure that a diff from baselines triggers a failed test (in debug only) if (SCREAM_ENABLE_BASELINE_TESTS) @@ -106,11 +112,3 @@ if (SCREAM_ENABLE_BASELINE_TESTS) EXCLUDE_MAIN_CPP ${FORCE_RUN_DIFF_FAILS}) endif() - -# Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it -if (SCREAM_TEST_MAX_THREADS GREATER 1) - # ECUT only adds _ompX if we have more than one value of X, or if X>1 - set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) -endif() - -set_tests_properties (p3_run_and_cmp${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") From d86865e814ca6031063ac0ef3aa9a7e7fa7e8946 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 14 Nov 2024 11:30:51 -0700 Subject: [PATCH 309/366] Fix threads arg --- components/eamxx/src/physics/p3/tests/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 9d2b2419c97..d705b30cc4e 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -55,11 +55,11 @@ if (SCREAM_ENABLE_BASELINE_TESTS) set(P3_THREADS "${SCREAM_TEST_MAX_THREADS}") else() set(BASELINE_FILE_ARG "-c -b ${SCREAM_BASELINES_DIR}/data") - set(P3_THREADS "1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}") + set(P3_THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}) endif() else() set(BASELINE_FILE_ARG "-n") # no baselines - set(P3_THREADS "1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}") + set(P3_THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}) endif() CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" From 6ce5b69e4a7fa02c2efb25efaade26e0827728ca Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 14 Nov 2024 14:05:19 -0700 Subject: [PATCH 310/366] Turn YAKL back on by default for now --- components/eamxx/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 5a7149234a5..07c3cda7672 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -212,8 +212,8 @@ endif() # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) -option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" FALSE) -option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" TRUE) +option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE) +option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE) if (SCREAM_RRTMGP_ENABLE_YAKL) add_definitions("-DRRTMGP_ENABLE_YAKL") endif() From f83702325bec771ec711bbe7c12f77dbeb88e40f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 14 Nov 2024 16:04:03 -0700 Subject: [PATCH 311/366] Remove some debugging stuff --- components/eamxx/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 9b6f54e5b05..07c3cda7672 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -200,9 +200,6 @@ option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them") set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") -message("JGF DEFAUL_SMALL_KERNELS is ${DEFAULT_SMALL_KERNELS}") -message("JGF SCREAM_SMALL_KERNELS is ${SCREAM_SMALL_KERNELS}") -message("JGF SCREAM_P3_SMALL_KERNELS is ${SCREAM_P3_SMALL_KERNELS}") set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_SHOC_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") From 03ab7123b68663846ecd738d512d3587b3a09226 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Thu, 14 Nov 2024 18:16:14 -0500 Subject: [PATCH 312/366] typo in links in eamxx docs --- components/eamxx/docs/developer/kokkos_ekat.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/docs/developer/kokkos_ekat.md b/components/eamxx/docs/developer/kokkos_ekat.md index 2432290a67a..a736384b2c6 100644 --- a/components/eamxx/docs/developer/kokkos_ekat.md +++ b/components/eamxx/docs/developer/kokkos_ekat.md @@ -56,8 +56,9 @@ where - `DataType`: scalar type of the view, given as `ScalarType`+`*`(x's number of run-time dimensions). E.g., a 2D view of doubles will have `DataType = double**`. There is also an ability to define compile-time dimensions by - using `[]`, see [Kokkos wiki section on views]( - wiki/API/core/view/view.html). + using `[]`, see + [Kokkos wiki section on views] + (). - `LayoutType`: mapping of indices into the underlying 1D memory storage. Types are: - `LayoutRight` (used in EAMxx): strides increase from the right most to the From 235493640b26533d6af8f812f23c8f1117ade2cd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 14 Nov 2024 11:11:56 -0700 Subject: [PATCH 313/366] EAMxx: change ghci-snl-cuda specs Grab number of gpus without hard-coding it --- components/eamxx/scripts/machines_specs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 41d09f4683e..66026191e05 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -221,9 +221,10 @@ class GHCISNLCuda(Machine): concrete = True @classmethod def setup(cls): - super().setup_base(name="ghci-snl-cuda",num_bld_res=16,num_run_res=1) + super().setup_base(name="ghci-snl-cuda") cls.baselines_dir = "/projects/e3sm/baselines/scream/ghci-snl-cuda" cls.gpu_arch = "cuda" + cls.num_run_res = int(run_cmd_no_fail("nvidia-smi --query-gpu=name --format=csv,noheader | wc -l")) ############################################################################### class Lassen(Machine): From 93a99aac5856d9dc3dab5d67d4a81c631e5f60e2 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 14 Nov 2024 11:12:18 -0700 Subject: [PATCH 314/366] Workflows: get Cuda arch from nvidia-smi in eamxx-sa-XYZ.yml --- .github/workflows/eamxx-sa-coverage.yml | 34 +++++++++++++++++++++++- .github/workflows/eamxx-sa-sanitizer.yml | 34 +++++++++++++++++++++++- .github/workflows/eamxx-sa-testing.yml | 34 +++++++++++++++++++++++- 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eamxx-sa-coverage.yml b/.github/workflows/eamxx-sa-coverage.yml index aa69d6263e6..1904b8e58df 100644 --- a/.github/workflows/eamxx-sa-coverage.yml +++ b/.github/workflows/eamxx-sa-coverage.yml @@ -48,6 +48,38 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger + - name: Get CUDA Arch + run: | + # Ensure nvidia-smi is available + if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed." + exit 1 + fi + + # Get the GPU model from nvidia-smi + gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) + case "$gpu_model" in + *"H100"*) + Hopper=ON + ARCH=90 + ;; + *"A100"*) + Amper=ON + ARCH=80 + ;; + *"V100"*) + Volta=ON + ARCH=70 + ;; + *) + echo "Unsupported GPU model: $gpu_model" + exit 1 + ;; + esac + + # Set the output variables for the next step + echo "KOKKOS_ARCH=${KOKKOS_ARCH}" >> $GITHUB_ENV + echo "CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}" >> $GITHUB_ENV - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -55,4 +87,4 @@ jobs: machine: ghci-snl-cuda generate: false submit: ${{ env.submit }} - cmake-configs: Kokkos_ARCH_VOLTA70=ON;CMAKE_CUDA_ARCHITECTURES=70 + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.ARCH }} diff --git a/.github/workflows/eamxx-sa-sanitizer.yml b/.github/workflows/eamxx-sa-sanitizer.yml index 7e3a1a49fcf..77b44d447f3 100644 --- a/.github/workflows/eamxx-sa-sanitizer.yml +++ b/.github/workflows/eamxx-sa-sanitizer.yml @@ -52,6 +52,38 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger + - name: Get CUDA Arch + run: | + # Ensure nvidia-smi is available + if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed." + exit 1 + fi + + # Get the GPU model from nvidia-smi + gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) + case "$gpu_model" in + *"H100"*) + Hopper=ON + ARCH=90 + ;; + *"A100"*) + Amper=ON + ARCH=80 + ;; + *"V100"*) + Volta=ON + ARCH=70 + ;; + *) + echo "Unsupported GPU model: $gpu_model" + exit 1 + ;; + esac + + # Set the output variables for the next step + echo "KOKKOS_ARCH=${KOKKOS_ARCH}" >> $GITHUB_ENV + echo "CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}" >> $GITHUB_ENV - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -59,4 +91,4 @@ jobs: machine: ghci-snl-cuda generate: false submit: ${{ env.submit }} - cmake-configs: Kokkos_ARCH_VOLTA70=ON;CMAKE_CUDA_ARCHITECTURES=70 + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.ARCH }} diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index baa1508ea6c..22b8e1e360f 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -172,6 +172,38 @@ jobs: echo "generate=true" >> $GITHUB_ENV fi fi + - name: Get CUDA Arch + run: | + # Ensure nvidia-smi is available + if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed." + exit 1 + fi + + # Get the GPU model from nvidia-smi + gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) + case "$gpu_model" in + *"H100"*) + Hopper=ON + ARCH=90 + ;; + *"A100"*) + Amper=ON + ARCH=80 + ;; + *"V100"*) + Volta=ON + ARCH=70 + ;; + *) + echo "Unsupported GPU model: $gpu_model" + exit 1 + ;; + esac + + # Set the output variables for the next step + echo "KOKKOS_ARCH=${KOKKOS_ARCH}" >> $GITHUB_ENV + echo "CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}" >> $GITHUB_ENV - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -179,4 +211,4 @@ jobs: machine: ghci-snl-cuda generate: ${{ env.generate }} submit: ${{ env.submit }} - cmake-configs: Kokkos_ARCH_VOLTA70=ON;CMAKE_CUDA_ARCHITECTURES=70 + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.ARCH }} From 4625182ff5d486311469fb12aa6ec2f933618f2b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 14 Nov 2024 21:59:07 -0700 Subject: [PATCH 315/366] Workflows: fetch changed files in chunks There are page size limits when using rest API --- .github/workflows/eamxx-sa-testing.yml | 19 ++++++++++++++++--- .github/workflows/eamxx-scripts-tests.yml | 19 ++++++++++++++++--- .github/workflows/eamxx-v1-testing.yml | 19 ++++++++++++++++--- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index baa1508ea6c..88cd5e17a9e 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -62,9 +62,22 @@ jobs: pattern=$(IFS=\|; echo "${paths[*]}") # Use the GitHub API to get the list of changed files - response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}/files") - changed_files=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g') + # There are page size limits, so do it in chunks + page=1 + while true; do + response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page") + + # Check if the response is empty, and break if it is + [ -z "$response" ] && break + + changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n' + + # Check if there are more pages, and quite if there aren't + [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break + + page=$((page + 1)) + done # Check for matches and echo the matching files (or "" if none) matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") diff --git a/.github/workflows/eamxx-scripts-tests.yml b/.github/workflows/eamxx-scripts-tests.yml index 8de453de387..52d856f8459 100644 --- a/.github/workflows/eamxx-scripts-tests.yml +++ b/.github/workflows/eamxx-scripts-tests.yml @@ -41,9 +41,22 @@ jobs: pattern=$(IFS=\|; echo "${paths[*]}") # Use the GitHub API to get the list of changed files - response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}/files") - changed_files=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g') + # There are page size limits, so do it in chunks + page=1 + while true; do + response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page") + + # Check if the response is empty, and break if it is + [ -z "$response" ] && break + + changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n' + + # Check if there are more pages, and quite if there aren't + [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break + + page=$((page + 1)) + done # Check for matches and echo the matching files (or "" if none) matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") diff --git a/.github/workflows/eamxx-v1-testing.yml b/.github/workflows/eamxx-v1-testing.yml index 43cc7623f88..d55ed8252a5 100644 --- a/.github/workflows/eamxx-v1-testing.yml +++ b/.github/workflows/eamxx-v1-testing.yml @@ -54,9 +54,22 @@ jobs: pattern=$(IFS=\|; echo "${paths[*]}") # Use the GitHub API to get the list of changed files - response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}/files") - changed_files=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g') + # There are page size limits, so do it in chunks + page=1 + while true; do + response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page") + + # Check if the response is empty, and break if it is + [ -z "$response" ] && break + + changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n' + + # Check if there are more pages, and quite if there aren't + [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break + + page=$((page + 1)) + done # Check for matches and echo the matching files (or "" if none) matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") From 0f2c4ba917473374f727e4a527d85005bf8cdc6e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 14 Nov 2024 22:18:55 -0700 Subject: [PATCH 316/366] Workflows: fix job name --- .github/workflows/eamxx-sa-sanitizer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eamxx-sa-sanitizer.yml b/.github/workflows/eamxx-sa-sanitizer.yml index 7e3a1a49fcf..14195e7ba3a 100644 --- a/.github/workflows/eamxx-sa-sanitizer.yml +++ b/.github/workflows/eamxx-sa-sanitizer.yml @@ -18,7 +18,7 @@ env: jobs: gcc-openmp: runs-on: [self-hosted, ghci-snl-cpu, gcc] - name: gcc-openmp / cov + name: gcc-openmp / valg steps: - name: Check out the repository uses: actions/checkout@v4 From 49d89f96e773cf7165f92c552d2afb6636dfac30 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 15 Nov 2024 10:16:41 -0700 Subject: [PATCH 317/366] Workflows: fix detection of CUDA arch in eamxx-sa workflows --- .github/workflows/eamxx-sa-coverage.yml | 19 ++++++++----------- .github/workflows/eamxx-sa-sanitizer.yml | 19 ++++++++----------- .github/workflows/eamxx-sa-testing.yml | 19 ++++++++----------- 3 files changed, 24 insertions(+), 33 deletions(-) diff --git a/.github/workflows/eamxx-sa-coverage.yml b/.github/workflows/eamxx-sa-coverage.yml index 1904b8e58df..6e767021c5e 100644 --- a/.github/workflows/eamxx-sa-coverage.yml +++ b/.github/workflows/eamxx-sa-coverage.yml @@ -56,30 +56,27 @@ jobs: exit 1 fi - # Get the GPU model from nvidia-smi + # Get the GPU model from nvidia-smi, and set env for next step gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) case "$gpu_model" in *"H100"*) - Hopper=ON + echo "Hopper=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=90" >> $GITHUB_ENV ARCH=90 ;; *"A100"*) - Amper=ON - ARCH=80 + echo "Ampere=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=80" >> $GITHUB_ENV ;; *"V100"*) - Volta=ON - ARCH=70 + echo "Volta=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=70" >> $GITHUB_ENV ;; *) echo "Unsupported GPU model: $gpu_model" exit 1 ;; esac - - # Set the output variables for the next step - echo "KOKKOS_ARCH=${KOKKOS_ARCH}" >> $GITHUB_ENV - echo "CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}" >> $GITHUB_ENV - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -87,4 +84,4 @@ jobs: machine: ghci-snl-cuda generate: false submit: ${{ env.submit }} - cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.ARCH }} + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }} diff --git a/.github/workflows/eamxx-sa-sanitizer.yml b/.github/workflows/eamxx-sa-sanitizer.yml index 77b44d447f3..2800716e88c 100644 --- a/.github/workflows/eamxx-sa-sanitizer.yml +++ b/.github/workflows/eamxx-sa-sanitizer.yml @@ -60,30 +60,27 @@ jobs: exit 1 fi - # Get the GPU model from nvidia-smi + # Get the GPU model from nvidia-smi, and set env for next step gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) case "$gpu_model" in *"H100"*) - Hopper=ON + echo "Hopper=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=90" >> $GITHUB_ENV ARCH=90 ;; *"A100"*) - Amper=ON - ARCH=80 + echo "Ampere=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=80" >> $GITHUB_ENV ;; *"V100"*) - Volta=ON - ARCH=70 + echo "Volta=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=70" >> $GITHUB_ENV ;; *) echo "Unsupported GPU model: $gpu_model" exit 1 ;; esac - - # Set the output variables for the next step - echo "KOKKOS_ARCH=${KOKKOS_ARCH}" >> $GITHUB_ENV - echo "CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}" >> $GITHUB_ENV - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -91,4 +88,4 @@ jobs: machine: ghci-snl-cuda generate: false submit: ${{ env.submit }} - cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.ARCH }} + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }} diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index 22b8e1e360f..901ba87fcd5 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -180,30 +180,27 @@ jobs: exit 1 fi - # Get the GPU model from nvidia-smi + # Get the GPU model from nvidia-smi, and set env for next step gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) case "$gpu_model" in *"H100"*) - Hopper=ON + echo "Hopper=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=90" >> $GITHUB_ENV ARCH=90 ;; *"A100"*) - Amper=ON - ARCH=80 + echo "Ampere=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=80" >> $GITHUB_ENV ;; *"V100"*) - Volta=ON - ARCH=70 + echo "Volta=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=70" >> $GITHUB_ENV ;; *) echo "Unsupported GPU model: $gpu_model" exit 1 ;; esac - - # Set the output variables for the next step - echo "KOKKOS_ARCH=${KOKKOS_ARCH}" >> $GITHUB_ENV - echo "CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}" >> $GITHUB_ENV - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -211,4 +208,4 @@ jobs: machine: ghci-snl-cuda generate: ${{ env.generate }} submit: ${{ env.submit }} - cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.ARCH }} + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }} From b2e02ea9f16b09cc11cc0bf7f45d91603aa21d6b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 15 Nov 2024 10:28:58 -0700 Subject: [PATCH 318/366] Workflows: allow to force cdash submission via workflow_dispatch for eamxx --- .github/workflows/eamxx-sa-coverage.yml | 8 +++++++- .github/workflows/eamxx-sa-sanitizer.yml | 8 +++++++- .github/workflows/eamxx-sa-testing.yml | 7 ++++++- .github/workflows/eamxx-scripts-tests.yml | 11 ++++++++++- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eamxx-sa-coverage.yml b/.github/workflows/eamxx-sa-coverage.yml index 6e767021c5e..46e87f8b8b0 100644 --- a/.github/workflows/eamxx-sa-coverage.yml +++ b/.github/workflows/eamxx-sa-coverage.yml @@ -2,6 +2,11 @@ name: eamxx-sa-coverage on: workflow_dispatch: + inputs: + submit: + description: 'Force cdash submission' + required: true + type: boolean # Add schedule trigger for nightly runs at midnight MT (Standard Time) schedule: @@ -13,7 +18,8 @@ concurrency: cancel-in-progress: true env: - submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }} # Submit to cdash only for nightlies + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} jobs: gcc-openmp: diff --git a/.github/workflows/eamxx-sa-sanitizer.yml b/.github/workflows/eamxx-sa-sanitizer.yml index 2800716e88c..7684f076341 100644 --- a/.github/workflows/eamxx-sa-sanitizer.yml +++ b/.github/workflows/eamxx-sa-sanitizer.yml @@ -2,6 +2,11 @@ name: eamxx-sa-sanitizer on: workflow_dispatch: + inputs: + submit: + description: 'Force cdash submission' + required: true + type: boolean # Add schedule trigger for nightly runs at midnight MT (Standard Time) schedule: @@ -13,7 +18,8 @@ concurrency: cancel-in-progress: true env: - submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }} # Submit to cdash only for nightlies + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} jobs: gcc-openmp: diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index 901ba87fcd5..831a9e6fc69 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -21,6 +21,10 @@ on: description: 'Generate baselines' required: true type: boolean + submit: + description: 'Force cdash submission' + required: true + type: boolean # Add schedule trigger for nightly runs at midnight MT (Standard Time) schedule: @@ -34,7 +38,8 @@ concurrency: cancel-in-progress: true env: - submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }} # Submit to cdash only for nightlies + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} jobs: pre_process_pr: diff --git a/.github/workflows/eamxx-scripts-tests.yml b/.github/workflows/eamxx-scripts-tests.yml index 8de453de387..860f75fa93c 100644 --- a/.github/workflows/eamxx-scripts-tests.yml +++ b/.github/workflows/eamxx-scripts-tests.yml @@ -8,6 +8,11 @@ on: # Manual run for debug purposes only workflow_dispatch: + inputs: + submit: + description: 'Force cdash submission' + required: true + type: boolean # Add schedule trigger for nightly runs at midnight MT (Standard Time) schedule: @@ -20,6 +25,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} + jobs: pre_process_pr: if: ${{ github.event_name == 'pull_request' }} @@ -83,7 +92,7 @@ jobs: - name: Run test run: | cd components/eamxx - if [ ${{ github.event_name == 'schedule' }} ]; then + if [ "${{ env.submit }}" == "true" ]; then ./scripts/scripts-ctest-driver -s -m ghci-snl-cpu else ./scripts/scripts-tests -f -m ghci-snl-cpu From 9673d6bddd699d8c5987a18720f8e5fea9309bf6 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Tue, 10 Sep 2024 01:17:58 -0600 Subject: [PATCH 319/366] emissions interface--running and calling mam4xx. next diagnose error --- ...and_online_emissions_process_interface.cpp | 131 ++++++++++++------ ...and_online_emissions_process_interface.hpp | 33 +++-- .../eamxx/src/physics/mam/online_emission.hpp | 102 ++++++++++++++ .../src/physics/mam/online_emission_impl.hpp | 60 ++++++++ .../single-process/mam/emissions/input.yaml | 28 ++-- 5 files changed, 293 insertions(+), 61 deletions(-) create mode 100644 components/eamxx/src/physics/mam/online_emission.hpp create mode 100644 components/eamxx/src/physics/mam/online_emission_impl.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 850a82d0896..7f85e931080 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -19,11 +19,11 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, // ================================================================ void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { - grid_ = grids_manager->get_grid("Physics"); + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; @@ -49,20 +49,21 @@ void MAMSrfOnlineEmiss::set_grids( //-------------------------------------------------------------------- srf_emiss_ dms; // File name, name and sectors - dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); dms.species_name = "dms"; - dms.sectors = {"DMS"}; - srf_emiss_species_.push_back(dms); // add to the vector + dms.sectors = {"DMS"}; + srf_emiss_species_.push_back(dms); // add to the vector //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- srf_emiss_ so2; // File name, name and sectors - so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); + so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); so2.species_name = "so2"; - so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; - srf_emiss_species_.push_back(so2); // add to the vector + so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(so2); // add to the vector + //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures //-------------------------------------------------------------------- @@ -70,8 +71,8 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors bc_a4.data_file = m_params.get("srf_emis_specifier_for_bc_a4"); bc_a4.species_name = "bc_a4"; - bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - srf_emiss_species_.push_back(bc_a4); // add to the vector + bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(bc_a4); // add to the vector //-------------------------------------------------------------------- // Init num_a1 srf emiss data structures @@ -80,9 +81,9 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors num_a1.data_file = m_params.get("srf_emis_specifier_for_num_a1"); num_a1.species_name = "num_a1"; - num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", - "num_a1_SO4_WST"}; - srf_emiss_species_.push_back(num_a1); // add to the vector + num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", + "num_a1_SO4_WST"}; + srf_emiss_species_.push_back(num_a1); // add to the vector //-------------------------------------------------------------------- // Init num_a2 srf emiss data structures @@ -91,8 +92,8 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors num_a2.data_file = m_params.get("srf_emis_specifier_for_num_a2"); num_a2.species_name = "num_a2"; - num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; - srf_emiss_species_.push_back(num_a2); // add to the vector + num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + srf_emiss_species_.push_back(num_a2); // add to the vector //-------------------------------------------------------------------- // Init num_a4 srf emiss data structures @@ -101,12 +102,12 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors num_a4.data_file = m_params.get("srf_emis_specifier_for_num_a4"); num_a4.species_name = "num_a4"; - num_a4.sectors = { - "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", - "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", - "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", - "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; - srf_emiss_species_.push_back(num_a4); // add to the vector + num_a4.sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + srf_emiss_species_.push_back(num_a4); // add to the vector //-------------------------------------------------------------------- // Init pom_a4 srf emiss data structures @@ -116,7 +117,7 @@ void MAMSrfOnlineEmiss::set_grids( pom_a4.data_file = m_params.get("srf_emis_specifier_for_pom_a4"); pom_a4.species_name = "pom_a4"; pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - srf_emiss_species_.push_back(pom_a4); // add to the vector + srf_emiss_species_.push_back(pom_a4); // add to the vector //-------------------------------------------------------------------- // Init so4_a1 srf emiss data structures @@ -125,7 +126,7 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors so4_a1.data_file = m_params.get("srf_emis_specifier_for_so4_a1"); so4_a1.species_name = "so4_a1"; - so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; + so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; srf_emiss_species_.push_back(so4_a1); //-------------------------------------------------------------------- @@ -135,21 +136,20 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors so4_a2.data_file = m_params.get("srf_emis_specifier_for_so4_a2"); so4_a2.species_name = "so4_a2"; - so4_a2.sectors = {"RCO", "TRA"}; + so4_a2.sectors = {"RCO", "TRA"}; srf_emiss_species_.push_back(so4_a2); //-------------------------------------------------------------------- // Init data structures to read and interpolate //-------------------------------------------------------------------- - for(srf_emiss_ &ispec_srf : srf_emiss_species_) { + for (srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::init_srf_emiss_objects( ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, // output ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, ispec_srf.data_out_, ispec_srf.dataReader_); } - -} // set_grid ends +} // set_grid ends // ================================================================ // REQUEST_BUFFER_SIZE_IN_BYTES @@ -167,9 +167,9 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG( - buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= + requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); @@ -206,18 +206,26 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { //-------------------------------------------------------------------- // Update surface emissions from file //-------------------------------------------------------------------- - for(srf_emiss_ &ispec_srf : srf_emiss_species_) { + for (srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::update_srfEmiss_data_from_file( ispec_srf.dataReader_, timestamp(), curr_month, *ispec_srf.horizInterp_, - ispec_srf.data_end_); // output + ispec_srf.data_end_); // output } + //-------------------------------------------------------------------- + // Initialize online emissions from file + //-------------------------------------------------------------------- + online_emis_data.init(ncol_); + onlineEmiss::init_from_input_file(m_params, online_emis_data); + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- preprocess_.initialize(constituent_fluxes_); + onlineEmiss::transfer_to_cflux(online_emis_data, spcIndex_in_pcnst_, + constituent_fluxes_); -} // end initialize_impl() +} // end initialize_impl() // ================================================================ // RUN_IMPL @@ -225,15 +233,60 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { void MAMSrfOnlineEmiss::run_impl(const double dt) { // Zero-out output Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); + // copy current values to online-emissions-local version + Kokkos::deep_copy(online_emis_data.constituent_fluxes, constituent_fluxes_); // Gather time and state information for interpolation auto ts = timestamp() + dt; + // Parallel loop over all the columns to update units + Kokkos::parallel_for( + "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + // need to initialize values for: + // SeasaltEmissionsData.{mpoly, mprot, mlip} + // DustEmissionsData.dust_dmt_vwr (done here for now) + // OnlineEmissionsData.{dust_flux_in, surface_temp, u_bottom, + // v_bottom, z_bottom, ocean_frac} + // NOTE: fortran mam4 gets dust_flux_in and ocean_frac from cam_in, + // which is a chunk-wise variable at this point + // (see: physpkg.F90:{1605 & 1327}) otherwise grabs the end/bottom + // col-value once inside aero_model_emissions() + + view_1d fluxes_col = Kokkos::subview(online_emis_data.constituent_fluxes, + icol, Kokkos::ALL()); + mam4::aero_model_emissions::aero_model_emissions(fluxes_col); + }); + + // for (onlineEmissData &ispec_online : online_emis_data) { + // //-------------------------------------------------------------------- + // // Modify units to MKS units (from molecules/cm2/s to kg/m2/s) + // //-------------------------------------------------------------------- + // // Get species index in array with pcnst dimension + // // (e.g., state_q or constituent_fluxes_) + // const int species_index = + // spcIndex_in_pcnst_.at(ispec_online.species_name); + + // // modify units from molecules/cm2/s to kg/m2/s + // auto fluxes_in_mks_units = this->fluxes_in_mks_units_; + // auto constituent_fluxes = this->constituent_fluxes_; + // const Real mfactor = + // amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; + // // Parallel loop over all the columns to update units + // Kokkos::parallel_for( + // "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + // fluxes_in_mks_units(icol) = + // ispec_online.data_out_.emiss_sectors(0, icol) * mfactor; + // constituent_fluxes(icol, species_index) = + // fluxes_in_mks_units(icol); + // }); + + // } // for loop for species + //-------------------------------------------------------------------- // Interpolate srf emiss data //-------------------------------------------------------------------- - for(srf_emiss_ &ispec_srf : srf_emiss_species_) { + for (srf_emiss_ &ispec_srf : srf_emiss_species_) { // Update TimeState, note the addition of dt ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); @@ -256,7 +309,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // modify units from molecules/cm2/s to kg/m2/s auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - auto constituent_fluxes = this->constituent_fluxes_; + auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units @@ -267,9 +320,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); }); - } // for loop for species + } // for loop for species Kokkos::fence(); -} // run_imple ends +} // run_impl ends // ============================================================================= -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 031fb62d8b7..65ac4f6375e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -6,6 +6,7 @@ // For MAM4 aerosol configuration #include +#include #include // For declaring surface and online emission class derived from atm process @@ -20,7 +21,7 @@ namespace scream { // The process responsible for handling MAM4 surface and online emissions. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { - using KT = ekat::KokkosTypes; + using KT = ekat::KokkosTypes; using view_1d = typename KT::template view_1d; using view_2d = typename KT::template view_2d; @@ -40,10 +41,12 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { view_1d fluxes_in_mks_units_; // Unified atomic mass unit used for unit conversion (BAD constant) - static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu + static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu - public: +public: using srfEmissFunc = mam_coupling::srfEmissFunctions; + using onlineEmiss = + mam_coupling::onlineEmissions; // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -59,8 +62,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::string name() const { return "mam_srf_online_emissions"; } // grid - void set_grids( - const std::shared_ptr grids_manager) override; + void + set_grids(const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -85,9 +88,9 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { } // local variables for preprocess struct view_2d constituent_fluxes_pre_; - }; // MAMSrfOnlineEmiss::Preprocess + }; // MAMSrfOnlineEmiss::Preprocess - private: +private: // preprocessing scratch pad Preprocess preprocess_; @@ -95,9 +98,11 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // FIXME: Remove the hardwired indices and use a function // to find them from an array. const std::map spcIndex_in_pcnst_ = { - {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, - {"num_a1", 22}, {"so4_a2", 23}, {"num_a2", 27}, - {"pom_a4", 36}, {"bc_a4", 37}, {"num_a4", 39}}; + {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, {"dst_a1", 19}, + {"ncl_a1", 20}, {"mom_a1", 21}, {"num_a1", 22}, {"so4_a2", 23}, + {"ncl_a2", 25}, {"mom_a2", 26}, {"num_a2", 27}, {"dst_a3", 28}, + {"ncl_a3", 29}, {"num_a3", 35}, {"pom_a4", 36}, {"bc_a4", 37}, + {"mom_a4", 38}, {"num_a4", 39}}; // A struct carrying all the fields needed to read // surface emissions of a species @@ -122,12 +127,14 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // A vector for carrying emissions for all the species std::vector srf_emiss_species_; + onlineEmiss::onlineEmissData online_emis_data; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; -}; // MAMSrfOnlineEmiss +}; // MAMSrfOnlineEmiss -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/online_emission.hpp b/components/eamxx/src/physics/mam/online_emission.hpp new file mode 100644 index 00000000000..0a2b3290e68 --- /dev/null +++ b/components/eamxx/src/physics/mam/online_emission.hpp @@ -0,0 +1,102 @@ +#ifndef ONLINE_EMISSION_HPP +#define ONLINE_EMISSION_HPP + +#include "share/util/scream_timing.hpp" + +namespace scream::mam_coupling { +namespace { + +template struct onlineEmissions { + using Device = DeviceType; + + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + + struct onlineEmissTimeState { + onlineEmissTimeState() = default; + // Whether the timestate has been initialized. + // The current month + int current_month = -1; + // Julian Date for the beginning of the month, as defined in + // /src/share/util/scream_time_stamp.hpp + // See this file for definition of Julian Date. + Real t_beg_month; + // Current simulation Julian Date + Real t_now; + // Number of days in the current month, cast as a Real + Real days_this_month; + }; // onlineEmissTimeState + + struct onlineEmissData { + // Basic spatial dimensions of the data + int ncols; + view_2d flux_data; + // local copy of main fluxes array + view_2d constituent_fluxes; + // FIXME: read this from input or get from mam4xx::aero_model_emissions? + const std::vector spec_names = {"so2", "soag", "bc_a4", + "num_a1", "num_a2", "num_a4", + "pom_a4", "so4_a1", "so4_a2"}; + // FIXME: change this when the above is dynamically-determined + int nspec = 9; + const std::string root_IC_str = "initial_condition_"; + + onlineEmissData() = default; + onlineEmissData(const int ncol_, const int nspec_) + : ncols(ncol_), nspec(nspec_) { + init(ncols, nspec, true); + } + + onlineEmissData(const int ncol_) : ncols(ncol_) { + init(ncols, nspec, true); + } + + void init(const int ncol_, const int nspec_, const bool allocate_) { + ncols = ncol_; + nspec = nspec_; + if (allocate_) + flux_data = view_2d("onlineEmissData", nspec, ncols); + } // onlineEmissData init + void init(const int ncol_, const bool allocate_) { + ncols = ncol_; + if (allocate_) + flux_data = view_2d("onlineEmissData", nspec, ncols); + } // onlineEmissData init + void init(const int ncol_) { + ncols = ncol_; + flux_data = view_2d("onlineEmissData", nspec, ncols); + } // onlineEmissData init + }; // onlineEmissData + + // The output is really just onlineEmissData, but for clarity it might + // help to see a onlineEmissOutput along a onlineEmissInput in functions + // signatures + // using onlineEmissOutput = onlineEmissData; + + // --------------------------------------------------------------------------- + // Online emissions routines + // --------------------------------------------------------------------------- + // FIXME: + // static void onlineEmiss_main(const onlineEmissTimeState &time_state, + // const onlineEmissInput &data_beg, + // const onlineEmissInput &data_end, + // const onlineEmissOutput &data_out); + + static void init_from_input_file(const ekat::ParameterList &m_params, + onlineEmissData &data); + static void + transfer_to_cflux(const onlineEmissData &data, + const std::map idx_map, + view_2d &fluxes); + // static void update_onlineEmiss_timestate( + // std::shared_ptr &scorpio_reader, + // const util::TimeStamp &ts, AbstractRemapper &onlineEmiss_horiz_interp, + // onlineEmissTimeState &time_state, onlineEmissInput &onlineEmiss_beg, + // onlineEmissInput &onlineEmiss_end); + +}; // struct onlineEmissions +} // namespace +} // namespace scream::mam_coupling +#endif // ONLINE_EMISSION_HPP + +#include "online_emission_impl.hpp" diff --git a/components/eamxx/src/physics/mam/online_emission_impl.hpp b/components/eamxx/src/physics/mam/online_emission_impl.hpp new file mode 100644 index 00000000000..5d374bfba76 --- /dev/null +++ b/components/eamxx/src/physics/mam/online_emission_impl.hpp @@ -0,0 +1,60 @@ +#ifndef ONLINE_EMISSION_IMPL_HPP +#define ONLINE_EMISSION_IMPL_HPP + +namespace scream::mam_coupling { +namespace { + +template +void onlineEmissions::init_from_input_file( + const ekat::ParameterList ¶ms, onlineEmissData &data) { + + const int nspec = data.nspec; + const int ncols = data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nspec); + + start_timer("EAMxx::onlineEmiss::init_onlineEmiss_data_from_input_file"); + // 1. Read from input file + // FIXME: currently reading a single placeholder scalar--should be + // ncols-sized array + for (int ispec = 0; ispec < nspec; ++ispec) { + Real init_cond_val = + params.get(data.root_IC_str + data.spec_names[ispec]); + // FIXME: is this overkill--i.e., would a mirror/deep_copy make more sense? + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int jcol = team.league_rank(); // column index + data.flux_data(ispec, jcol) = init_cond_val; + }); + } + stop_timer("EAMxx::onlineEmiss::init_onlineEmiss_data_from_input_file"); + +} // end init_from_input_file() + +template +void onlineEmissions::transfer_to_cflux( + const onlineEmissData &data, const std::map idx_map, + view_2d &fluxes) { + // FIXME: move out to onlineEmissions struct? + const int nspec = data.nspec; + const int ncols = data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nspec); + + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int jcol = team.league_rank(); // column index + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, nspec), [&](const int ispec) { + auto s_idx = idx_map.at(data.spec_names[ispec]); + fluxes(jcol, s_idx) = data.flux_data(ispec, jcol); + }); + }); +} + +} // namespace +} // namespace scream::mam_coupling + +#endif // ONLINE_EMISSION_IMPL_HPP diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index e7af45178aa..3c7bd352f72 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,15 +13,25 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + # FIXME: just set everything to a constant scalar for testing + initial_condition_so2: 1.0 + initial_condition_soag: 2.0 + initial_condition_bc_a4: 3.0 + initial_condition_num_a1: 4.0 + initial_condition_num_a2: 5.0 + initial_condition_num_a4: 6.0 + initial_condition_pom_a4: 7.0 + initial_condition_so4_a1: 8.0 + initial_condition_so4_a2: 9.0 grids_manager: Type: Mesh Free From c3f2ebff570a2ab8bcc576fbcf92eaba8c621e2c Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 9 Oct 2024 15:47:02 -0600 Subject: [PATCH 320/366] cleanup to fix warnings and change path to emissions input files --- .../eamxx/src/physics/mam/online_emission.hpp | 9 -------- .../src/physics/mam/online_emission_impl.hpp | 4 ---- .../eamxx/src/physics/mam/srf_emission.hpp | 3 --- .../src/physics/mam/srf_emission_impl.hpp | 21 ++++++++++++------- .../single-process/mam/emissions/input.yaml | 18 ++++++++-------- 5 files changed, 22 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/physics/mam/online_emission.hpp b/components/eamxx/src/physics/mam/online_emission.hpp index 0a2b3290e68..bc5a510ccf7 100644 --- a/components/eamxx/src/physics/mam/online_emission.hpp +++ b/components/eamxx/src/physics/mam/online_emission.hpp @@ -4,8 +4,6 @@ #include "share/util/scream_timing.hpp" namespace scream::mam_coupling { -namespace { - template struct onlineEmissions { using Device = DeviceType; @@ -88,14 +86,7 @@ template struct onlineEmissions { transfer_to_cflux(const onlineEmissData &data, const std::map idx_map, view_2d &fluxes); - // static void update_onlineEmiss_timestate( - // std::shared_ptr &scorpio_reader, - // const util::TimeStamp &ts, AbstractRemapper &onlineEmiss_horiz_interp, - // onlineEmissTimeState &time_state, onlineEmissInput &onlineEmiss_beg, - // onlineEmissInput &onlineEmiss_end); - }; // struct onlineEmissions -} // namespace } // namespace scream::mam_coupling #endif // ONLINE_EMISSION_HPP diff --git a/components/eamxx/src/physics/mam/online_emission_impl.hpp b/components/eamxx/src/physics/mam/online_emission_impl.hpp index 5d374bfba76..860a4cac2be 100644 --- a/components/eamxx/src/physics/mam/online_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/online_emission_impl.hpp @@ -2,8 +2,6 @@ #define ONLINE_EMISSION_IMPL_HPP namespace scream::mam_coupling { -namespace { - template void onlineEmissions::init_from_input_file( const ekat::ParameterList ¶ms, onlineEmissData &data) { @@ -53,8 +51,6 @@ void onlineEmissions::transfer_to_cflux( }); }); } - -} // namespace } // namespace scream::mam_coupling #endif // ONLINE_EMISSION_IMPL_HPP diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 29aaca421ea..8dc5be1d05a 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -4,8 +4,6 @@ #include "share/util/scream_timing.hpp" namespace scream::mam_coupling { -namespace { - template struct srfEmissFunctions { using Device = DeviceType; @@ -131,7 +129,6 @@ struct srfEmissFunctions { std::shared_ptr &SrfEmissDataReader); }; // struct srfEmissFunctions -} // namespace } // namespace scream::mam_coupling #endif // SRF_EMISSION_HPP diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 48dc1fa7087..b090e3746cb 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -6,8 +6,6 @@ #include "share/io/scream_scorpio_interface.hpp" namespace scream::mam_coupling { -namespace { - template std::shared_ptr srfEmissFunctions::create_horiz_remapper( @@ -104,6 +102,9 @@ void srfEmissFunctions::perform_time_interpolation( // NOTE: we *assume* data_beg and data_end have the *same* hybrid v coords. // IF this ever ceases to be the case, you can interp those too. + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + // Gather time stamp info auto &t_now = time_state.t_now; auto &t_beg = time_state.t_beg_month; @@ -181,6 +182,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int time_index, // zero-based AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input) { using namespace ShortFieldTagsNames; + // NOTE: these are currently unused + // using ESU = ekat::ExeSpaceUtils; + // using Member = typename KokkosTypes::MemberType; start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); @@ -201,10 +205,13 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) - .get_header() - .get_identifier() - .get_layout(); + // NOTE: these are currently unused + // const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) + // .get_header() + // .get_identifier() + // .get_layout(); + + const int ncols = layout.dim(COL); // Read fields from the file for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { @@ -279,8 +286,6 @@ void srfEmissFunctions::init_srf_emiss_objects( SrfEmissDataReader = create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); } // init_srf_emiss_objects - -} // namespace } // namespace scream::mam_coupling #endif // SRF_EMISSION_IMPL_HPP diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 3c7bd352f72..93f187e7de3 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,15 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc # FIXME: just set everything to a constant scalar for testing initial_condition_so2: 1.0 initial_condition_soag: 2.0 From 749f2076bf667c874f9d6c541b0b44e839f3a464 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Tue, 15 Oct 2024 06:59:46 -0600 Subject: [PATCH 321/366] got online emis. interface working --- ...and_online_emissions_process_interface.cpp | 78 +++++++------------ ...and_online_emissions_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/online_emission.hpp | 42 ++++++---- .../src/physics/mam/online_emission_impl.hpp | 19 +++-- .../single-process/mam/emissions/input.yaml | 21 ++--- 5 files changed, 79 insertions(+), 83 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 7f85e931080..b50da0d4445 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -33,6 +33,9 @@ void MAMSrfOnlineEmiss::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process + // FIXME: check into whether this should be an "Updated" field because + // I haven't yet determined if an updated field needs to be output + // as a tendency, and thus the flux would need dt [s] factored out // ------------------------------------------------------------- static constexpr Units m2(m * m, "m2"); // Constituent fluxes of species in [kg/m2/s] @@ -215,15 +218,13 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { //-------------------------------------------------------------------- // Initialize online emissions from file //-------------------------------------------------------------------- - online_emis_data.init(ncol_); - onlineEmiss::init_from_input_file(m_params, online_emis_data); + online_emissions.online_emis_data.init(ncol_); + online_emissions.init_from_input_file(m_params); //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- preprocess_.initialize(constituent_fluxes_); - onlineEmiss::transfer_to_cflux(online_emis_data, spcIndex_in_pcnst_, - constituent_fluxes_); } // end initialize_impl() @@ -233,55 +234,10 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { void MAMSrfOnlineEmiss::run_impl(const double dt) { // Zero-out output Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); - // copy current values to online-emissions-local version - Kokkos::deep_copy(online_emis_data.constituent_fluxes, constituent_fluxes_); // Gather time and state information for interpolation auto ts = timestamp() + dt; - // Parallel loop over all the columns to update units - Kokkos::parallel_for( - "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - // need to initialize values for: - // SeasaltEmissionsData.{mpoly, mprot, mlip} - // DustEmissionsData.dust_dmt_vwr (done here for now) - // OnlineEmissionsData.{dust_flux_in, surface_temp, u_bottom, - // v_bottom, z_bottom, ocean_frac} - // NOTE: fortran mam4 gets dust_flux_in and ocean_frac from cam_in, - // which is a chunk-wise variable at this point - // (see: physpkg.F90:{1605 & 1327}) otherwise grabs the end/bottom - // col-value once inside aero_model_emissions() - - view_1d fluxes_col = Kokkos::subview(online_emis_data.constituent_fluxes, - icol, Kokkos::ALL()); - mam4::aero_model_emissions::aero_model_emissions(fluxes_col); - }); - - // for (onlineEmissData &ispec_online : online_emis_data) { - // //-------------------------------------------------------------------- - // // Modify units to MKS units (from molecules/cm2/s to kg/m2/s) - // //-------------------------------------------------------------------- - // // Get species index in array with pcnst dimension - // // (e.g., state_q or constituent_fluxes_) - // const int species_index = - // spcIndex_in_pcnst_.at(ispec_online.species_name); - - // // modify units from molecules/cm2/s to kg/m2/s - // auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - // auto constituent_fluxes = this->constituent_fluxes_; - // const Real mfactor = - // amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; - // // Parallel loop over all the columns to update units - // Kokkos::parallel_for( - // "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - // fluxes_in_mks_units(icol) = - // ispec_online.data_out_.emiss_sectors(0, icol) * mfactor; - // constituent_fluxes(icol, species_index) = - // fluxes_in_mks_units(icol); - // }); - - // } // for loop for species - //-------------------------------------------------------------------- // Interpolate srf emiss data //-------------------------------------------------------------------- @@ -314,13 +270,35 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units Kokkos::parallel_for( - "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + "srf_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { fluxes_in_mks_units(icol) = ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); }); } // for loop for species + + // TODO: check that units are consistent with srf emissions! + // copy current values to online-emissions-local version + Kokkos::deep_copy(online_emis_data.cfluxes, constituent_fluxes_); + // TODO: potentially combine with above parfor(icol) loop? + Kokkos::parallel_for( + "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + // need to initialize values for: + // SeasaltEmissionsData.{mpoly, mprot, mlip} + // DustEmissionsData.dust_dmt_vwr + // OnlineEmissionsData.{dust_flux_in, surface_temp, u_bottom, + // v_bottom, z_bottom, ocean_frac} + // NOTE: fortran mam4 gets dust_flux_in and ocean_frac from cam_in, + // which is a chunk-wise variable at this point + // (see: physpkg.F90:{1605 & 1327}) otherwise grabs the end/bottom + // col-value once inside aero_model_emissions() + + view_1d fluxes_col = Kokkos::subview( + online_emissions.online_emis_data.cfluxes, icol, Kokkos::ALL()); + mam4::aero_model_emissions::aero_model_emissions(fluxes_col); + }); + Kokkos::deep_copy(constituent_fluxes_, online_emis_data.cfluxes); Kokkos::fence(); } // run_impl ends diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 65ac4f6375e..8bfee35f2d9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -127,7 +127,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // A vector for carrying emissions for all the species std::vector srf_emiss_species_; - onlineEmiss::onlineEmissData online_emis_data; + onlineEmiss online_emissions; // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = diff --git a/components/eamxx/src/physics/mam/online_emission.hpp b/components/eamxx/src/physics/mam/online_emission.hpp index bc5a510ccf7..9841c874f85 100644 --- a/components/eamxx/src/physics/mam/online_emission.hpp +++ b/components/eamxx/src/physics/mam/online_emission.hpp @@ -9,6 +9,7 @@ template struct onlineEmissions { using KT = KokkosTypes; using MemberType = typename KT::MemberType; + static constexpr int pcnst = mam4::aero_model::pcnst; struct onlineEmissTimeState { onlineEmissTimeState() = default; @@ -30,14 +31,14 @@ template struct onlineEmissions { int ncols; view_2d flux_data; // local copy of main fluxes array - view_2d constituent_fluxes; + view_2d cfluxes; // FIXME: read this from input or get from mam4xx::aero_model_emissions? - const std::vector spec_names = {"so2", "soag", "bc_a4", - "num_a1", "num_a2", "num_a4", - "pom_a4", "so4_a1", "so4_a2"}; + const std::vector spec_names = { + "ncl_a1", "ncl_a2", "ncl_a3", "mom_a1", "mom_a2", "mom_a4", + "num_a1", "num_a2", "num_a3", "num_a4", "dst_a1", "dst_a3"}; // FIXME: change this when the above is dynamically-determined - int nspec = 9; - const std::string root_IC_str = "initial_condition_"; + int nspec = spec_names.size(); + const std::string root_IC_str = "online_emis_IC_"; onlineEmissData() = default; onlineEmissData(const int ncol_, const int nspec_) @@ -52,22 +53,29 @@ template struct onlineEmissions { void init(const int ncol_, const int nspec_, const bool allocate_) { ncols = ncol_; nspec = nspec_; - if (allocate_) + if (allocate_) { flux_data = view_2d("onlineEmissData", nspec, ncols); + cfluxes = view_2d("onlineEmisLocalCflux", ncols, pcnst); + } } // onlineEmissData init void init(const int ncol_, const bool allocate_) { ncols = ncol_; - if (allocate_) + if (allocate_) { flux_data = view_2d("onlineEmissData", nspec, ncols); + cfluxes = view_2d("onlineEmisLocalCflux", ncols, pcnst); + } } // onlineEmissData init void init(const int ncol_) { ncols = ncol_; flux_data = view_2d("onlineEmissData", nspec, ncols); + cfluxes = view_2d("onlineEmisLocalCflux", ncols, pcnst); } // onlineEmissData init }; // onlineEmissData + onlineEmissData online_emis_data; + // The output is really just onlineEmissData, but for clarity it might - // help to see a onlineEmissOutput along a onlineEmissInput in functions + // help to see a onlineEmissOutput along with onlineEmissInput in functions // signatures // using onlineEmissOutput = onlineEmissData; @@ -80,12 +88,16 @@ template struct onlineEmissions { // const onlineEmissInput &data_end, // const onlineEmissOutput &data_out); - static void init_from_input_file(const ekat::ParameterList &m_params, - onlineEmissData &data); - static void - transfer_to_cflux(const onlineEmissData &data, - const std::map idx_map, - view_2d &fluxes); + void init_from_input_file(const ekat::ParameterList &m_params); + void transfer_to_cflux(const onlineEmissData &data, + const std::map idx_map, + view_2d &fluxes); + // static void update_onlineEmiss_timestate( + // std::shared_ptr &scorpio_reader, + // const util::TimeStamp &ts, AbstractRemapper &onlineEmiss_horiz_interp, + // onlineEmissTimeState &time_state, onlineEmissInput &onlineEmiss_beg, + // onlineEmissInput &onlineEmiss_end); + }; // struct onlineEmissions } // namespace scream::mam_coupling #endif // ONLINE_EMISSION_HPP diff --git a/components/eamxx/src/physics/mam/online_emission_impl.hpp b/components/eamxx/src/physics/mam/online_emission_impl.hpp index 860a4cac2be..ec415c036a3 100644 --- a/components/eamxx/src/physics/mam/online_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/online_emission_impl.hpp @@ -3,11 +3,9 @@ namespace scream::mam_coupling { template -void onlineEmissions::init_from_input_file( - const ekat::ParameterList ¶ms, onlineEmissData &data) { - - const int nspec = data.nspec; - const int ncols = data.ncols; +void onlineEmissions::init_from_input_file(const ekat::ParameterList ¶ms) { + const int nspec = online_emis_data.nspec; + const int ncols = online_emis_data.ncols; using ExeSpace = typename KT::ExeSpace; using ESU = ekat::ExeSpaceUtils; const auto policy = ESU::get_default_team_policy(ncols, nspec); @@ -18,12 +16,12 @@ void onlineEmissions::init_from_input_file( // ncols-sized array for (int ispec = 0; ispec < nspec; ++ispec) { Real init_cond_val = - params.get(data.root_IC_str + data.spec_names[ispec]); + params.get(online_emis_data.root_IC_str + online_emis_data.spec_names[ispec]); // FIXME: is this overkill--i.e., would a mirror/deep_copy make more sense? Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MemberType &team) { const int jcol = team.league_rank(); // column index - data.flux_data(ispec, jcol) = init_cond_val; + online_emis_data.flux_data(ispec, jcol) = init_cond_val; }); } stop_timer("EAMxx::onlineEmiss::init_onlineEmiss_data_from_input_file"); @@ -47,9 +45,14 @@ void onlineEmissions::transfer_to_cflux( Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nspec), [&](const int ispec) { auto s_idx = idx_map.at(data.spec_names[ispec]); - fluxes(jcol, s_idx) = data.flux_data(ispec, jcol); + data.cfluxes(jcol, s_idx) = data.flux_data(ispec, jcol); + std::cout << "=========================================" << "\n"; + std::cout << "data.cfluxes(jcol, s_idx) = " << data.cfluxes(jcol, s_idx) << "\n"; + std::cout << "=========================================" << "\n"; }); }); + + Kokkos::deep_copy(fluxes, data.cfluxes); } } // namespace scream::mam_coupling diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 93f187e7de3..589f08fba47 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -23,15 +23,18 @@ atmosphere_processes: srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc # FIXME: just set everything to a constant scalar for testing - initial_condition_so2: 1.0 - initial_condition_soag: 2.0 - initial_condition_bc_a4: 3.0 - initial_condition_num_a1: 4.0 - initial_condition_num_a2: 5.0 - initial_condition_num_a4: 6.0 - initial_condition_pom_a4: 7.0 - initial_condition_so4_a1: 8.0 - initial_condition_so4_a2: 9.0 + online_emis_IC_ncl_a1: 1.0 + online_emis_IC_ncl_a2: 2.0 + online_emis_IC_ncl_a3: 3.0 + online_emis_IC_mom_a1: 4.0 + online_emis_IC_mom_a2: 5.0 + online_emis_IC_mom_a4: 6.0 + online_emis_IC_num_a1: 7.0 + online_emis_IC_num_a2: 8.0 + online_emis_IC_num_a3: 9.0 + online_emis_IC_num_a4: 10.0 + online_emis_IC_dst_a1: 11.0 + online_emis_IC_dst_a3: 12.0 grids_manager: Type: Mesh Free From 484d2c17d009dbb6eb96f1a3c0a538d7fac47930 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 16 Oct 2024 16:42:59 -0600 Subject: [PATCH 322/366] online emissions interface appears to be working --- ...and_online_emissions_process_interface.cpp | 28 +++++++------ ...and_online_emissions_process_interface.hpp | 4 +- .../eamxx/src/physics/mam/online_emission.hpp | 39 ++----------------- .../src/physics/mam/online_emission_impl.hpp | 17 +++----- .../single-process/mam/emissions/input.yaml | 38 ++++++++++++------ externals/mam4xx | 2 +- 6 files changed, 54 insertions(+), 74 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index b50da0d4445..1db0535cce8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -33,9 +33,13 @@ void MAMSrfOnlineEmiss::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process - // FIXME: check into whether this should be an "Updated" field because - // I haven't yet determined if an updated field needs to be output - // as a tendency, and thus the flux would need dt [s] factored out + // FIXME: this should likely be an updated field, since online emissisons + // expects input values for constituent_fluxes + // NOTE: the other option is that we do something like: + // add_field("constituent_fluxes_input", scalar2d_pcnct, + // kg / m2 / s, grid_name); + // and then bundle the online emissions computations as tendencies into the + // Computed constituent_fluxes field // ------------------------------------------------------------- static constexpr Units m2(m * m, "m2"); // Constituent fluxes of species in [kg/m2/s] @@ -275,16 +279,19 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); }); - } // for loop for species + auto &online_data = online_emissions.online_emis_data; // TODO: check that units are consistent with srf emissions! // copy current values to online-emissions-local version - Kokkos::deep_copy(online_emis_data.cfluxes, constituent_fluxes_); + Kokkos::deep_copy(online_data.cfluxes, constituent_fluxes_); // TODO: potentially combine with above parfor(icol) loop? Kokkos::parallel_for( "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - // need to initialize values for: + // NOTE: calling aero_model_emissions() this way hides the fact that + // some hard-coded data is initialized within mam4::aero_model_emissions + // some of these values definitely need to be read from here column-wise + // the structs.{values} holding the above are: // SeasaltEmissionsData.{mpoly, mprot, mlip} // DustEmissionsData.dust_dmt_vwr // OnlineEmissionsData.{dust_flux_in, surface_temp, u_bottom, @@ -293,14 +300,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // which is a chunk-wise variable at this point // (see: physpkg.F90:{1605 & 1327}) otherwise grabs the end/bottom // col-value once inside aero_model_emissions() - - view_1d fluxes_col = Kokkos::subview( - online_emissions.online_emis_data.cfluxes, icol, Kokkos::ALL()); + view_1d fluxes_col = Kokkos::subview(online_data.cfluxes, icol, Kokkos::ALL()); mam4::aero_model_emissions::aero_model_emissions(fluxes_col); }); - Kokkos::deep_copy(constituent_fluxes_, online_emis_data.cfluxes); + // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes + // in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert + Kokkos::deep_copy(constituent_fluxes_, online_data.cfluxes); Kokkos::fence(); } // run_impl ends - // ============================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 8bfee35f2d9..80c2efcccd7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -13,7 +13,6 @@ // class #include -// #include #include namespace scream { @@ -45,8 +44,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { public: using srfEmissFunc = mam_coupling::srfEmissFunctions; - using onlineEmiss = - mam_coupling::onlineEmissions; + using onlineEmiss = mam_coupling::onlineEmissions; // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); diff --git a/components/eamxx/src/physics/mam/online_emission.hpp b/components/eamxx/src/physics/mam/online_emission.hpp index 9841c874f85..b844f5b18e6 100644 --- a/components/eamxx/src/physics/mam/online_emission.hpp +++ b/components/eamxx/src/physics/mam/online_emission.hpp @@ -6,37 +6,21 @@ namespace scream::mam_coupling { template struct onlineEmissions { using Device = DeviceType; - using KT = KokkosTypes; using MemberType = typename KT::MemberType; static constexpr int pcnst = mam4::aero_model::pcnst; - struct onlineEmissTimeState { - onlineEmissTimeState() = default; - // Whether the timestate has been initialized. - // The current month - int current_month = -1; - // Julian Date for the beginning of the month, as defined in - // /src/share/util/scream_time_stamp.hpp - // See this file for definition of Julian Date. - Real t_beg_month; - // Current simulation Julian Date - Real t_now; - // Number of days in the current month, cast as a Real - Real days_this_month; - }; // onlineEmissTimeState - struct onlineEmissData { // Basic spatial dimensions of the data int ncols; view_2d flux_data; // local copy of main fluxes array view_2d cfluxes; - // FIXME: read this from input or get from mam4xx::aero_model_emissions? + // FIXME: read this from elsewhere? input? const std::vector spec_names = { "ncl_a1", "ncl_a2", "ncl_a3", "mom_a1", "mom_a2", "mom_a4", "num_a1", "num_a2", "num_a3", "num_a4", "dst_a1", "dst_a3"}; - // FIXME: change this when the above is dynamically-determined + // FIXME: change this when/if the above is dynamically-determined int nspec = spec_names.size(); const std::string root_IC_str = "online_emis_IC_"; @@ -50,6 +34,8 @@ template struct onlineEmissions { init(ncols, nspec, true); } + // overloads of init() in case npsec is not hard-coded, or if you want to + // control allocation via bool flag void init(const int ncol_, const int nspec_, const bool allocate_) { ncols = ncol_; nspec = nspec_; @@ -74,30 +60,13 @@ template struct onlineEmissions { onlineEmissData online_emis_data; - // The output is really just onlineEmissData, but for clarity it might - // help to see a onlineEmissOutput along with onlineEmissInput in functions - // signatures - // using onlineEmissOutput = onlineEmissData; - // --------------------------------------------------------------------------- // Online emissions routines // --------------------------------------------------------------------------- - // FIXME: - // static void onlineEmiss_main(const onlineEmissTimeState &time_state, - // const onlineEmissInput &data_beg, - // const onlineEmissInput &data_end, - // const onlineEmissOutput &data_out); - void init_from_input_file(const ekat::ParameterList &m_params); void transfer_to_cflux(const onlineEmissData &data, const std::map idx_map, view_2d &fluxes); - // static void update_onlineEmiss_timestate( - // std::shared_ptr &scorpio_reader, - // const util::TimeStamp &ts, AbstractRemapper &onlineEmiss_horiz_interp, - // onlineEmissTimeState &time_state, onlineEmissInput &onlineEmiss_beg, - // onlineEmissInput &onlineEmiss_end); - }; // struct onlineEmissions } // namespace scream::mam_coupling #endif // ONLINE_EMISSION_HPP diff --git a/components/eamxx/src/physics/mam/online_emission_impl.hpp b/components/eamxx/src/physics/mam/online_emission_impl.hpp index ec415c036a3..1f0ac91a854 100644 --- a/components/eamxx/src/physics/mam/online_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/online_emission_impl.hpp @@ -4,35 +4,32 @@ namespace scream::mam_coupling { template void onlineEmissions::init_from_input_file(const ekat::ParameterList ¶ms) { + // FIXME: move this out to the onlineEmissions struct to avoid extra work + // by doing it again below? const int nspec = online_emis_data.nspec; const int ncols = online_emis_data.ncols; using ExeSpace = typename KT::ExeSpace; using ESU = ekat::ExeSpaceUtils; const auto policy = ESU::get_default_team_policy(ncols, nspec); - - start_timer("EAMxx::onlineEmiss::init_onlineEmiss_data_from_input_file"); - // 1. Read from input file + // Read from input file // FIXME: currently reading a single placeholder scalar--should be - // ncols-sized array + // ncols-sized array when we know what the input data looks like for (int ispec = 0; ispec < nspec; ++ispec) { Real init_cond_val = params.get(online_emis_data.root_IC_str + online_emis_data.spec_names[ispec]); - // FIXME: is this overkill--i.e., would a mirror/deep_copy make more sense? + // TODO: is this overkill?--i.e., would a mirror/deep_copy make more sense? Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MemberType &team) { const int jcol = team.league_rank(); // column index online_emis_data.flux_data(ispec, jcol) = init_cond_val; }); } - stop_timer("EAMxx::onlineEmiss::init_onlineEmiss_data_from_input_file"); - } // end init_from_input_file() template void onlineEmissions::transfer_to_cflux( const onlineEmissData &data, const std::map idx_map, view_2d &fluxes) { - // FIXME: move out to onlineEmissions struct? const int nspec = data.nspec; const int ncols = data.ncols; using ExeSpace = typename KT::ExeSpace; @@ -46,12 +43,8 @@ void onlineEmissions::transfer_to_cflux( Kokkos::TeamThreadRange(team, nspec), [&](const int ispec) { auto s_idx = idx_map.at(data.spec_names[ispec]); data.cfluxes(jcol, s_idx) = data.flux_data(ispec, jcol); - std::cout << "=========================================" << "\n"; - std::cout << "data.cfluxes(jcol, s_idx) = " << data.cfluxes(jcol, s_idx) << "\n"; - std::cout << "=========================================" << "\n"; }); }); - Kokkos::deep_copy(fluxes, data.cfluxes); } } // namespace scream::mam_coupling diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 589f08fba47..7a3dfb43f22 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -23,18 +23,32 @@ atmosphere_processes: srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc # FIXME: just set everything to a constant scalar for testing - online_emis_IC_ncl_a1: 1.0 - online_emis_IC_ncl_a2: 2.0 - online_emis_IC_ncl_a3: 3.0 - online_emis_IC_mom_a1: 4.0 - online_emis_IC_mom_a2: 5.0 - online_emis_IC_mom_a4: 6.0 - online_emis_IC_num_a1: 7.0 - online_emis_IC_num_a2: 8.0 - online_emis_IC_num_a3: 9.0 - online_emis_IC_num_a4: 10.0 - online_emis_IC_dst_a1: 11.0 - online_emis_IC_dst_a3: 12.0 + online_emis_IC_ncl_a1: 0.0 + online_emis_IC_ncl_a2: 0.0 + online_emis_IC_ncl_a3: 0.0 + online_emis_IC_mom_a1: 0.16838301005552275E-013 + online_emis_IC_mom_a2: 0.26554873160224250E-016 + online_emis_IC_mom_a4: 0.0 + online_emis_IC_num_a1: 0.0 + online_emis_IC_num_a2: 0.0 + online_emis_IC_num_a3: 0.0 + online_emis_IC_num_a4: 0.0 + online_emis_IC_dst_a1: 0.0 + online_emis_IC_dst_a3: 0.18720550166902007E+003 + +# NOTE: these are the single-call results from the mam validation test +# (20) ncl_a1: 2.659900637e-13 +# (25) ncl_a2: 0 +# (29) ncl_a3: 2.391492482e-11 +# (21) mom_a1: 1.683830101e-14 +# (26) mom_a2: 2.655487316e-17 +# (38) mom_a4: 0 +# (22) num_a1: 2.594942753e-15 +# (27) num_a2: 4.092359179e-18 +# (35) num_a3: 0 +# (39) num_a4: 0 +# (19) dst_a1: 0 +# (28) dst_a3: 403.2611563 grids_manager: Type: Mesh Free diff --git a/externals/mam4xx b/externals/mam4xx index 4431bbd1eef..c537bab2695 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 4431bbd1eef46de25be3a04e7091c9255bd0b819 +Subproject commit c537bab2695af76290cc0b82462393761b5820ed From 829c560df200b3c7dcd7ef012e557948be909f6f Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 16 Oct 2024 17:34:24 -0600 Subject: [PATCH 323/366] bump mam4xx submodule, add comments to srf_online_emis_interface --- ..._mam_srf_and_online_emissions_process_interface.cpp | 10 ++++++++++ externals/mam4xx | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 1db0535cce8..c84bb803440 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -30,6 +30,16 @@ void MAMSrfOnlineEmiss::set_grids( static constexpr int pcnst = mam4::aero_model::pcnst; const FieldLayout scalar2d_pcnct = grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + // auto vector3d_mid = grid_->get_3d_vector_layout(true, 2); + + // FIXME: online emissions requires the following quantities for the + // OnlineEmissionsData struct: {surface_temp, u_bottom, v_bottom, z_bottom, + // ocean_frac} + // // Temperature + // add_field("T_mid", scalar3d_layout_mid, K, grid_name); + // add_field("horiz_winds", vector3d_mid, m/s, grid_name); + // vertical wind? + // ocean_frac? // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process diff --git a/externals/mam4xx b/externals/mam4xx index c537bab2695..67c309117ab 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit c537bab2695af76290cc0b82462393761b5820ed +Subproject commit 67c309117abe0be649d085c7aeaeb1e949f6b550 From b2dbb85c4b7bee72cd88b26f16bbbed936479aaa Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 16 Oct 2024 20:43:48 -0600 Subject: [PATCH 324/366] update mam4xx to main --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 67c309117ab..6bc2697f84b 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 67c309117abe0be649d085c7aeaeb1e949f6b550 +Subproject commit 6bc2697f84b38503cc8fe638a6281ea107180e40 From 24d3e11c04506a572e04de5718d048e531754401 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Mon, 21 Oct 2024 13:00:52 -0600 Subject: [PATCH 325/366] comment out error-inducing code in mam4_amicphys.cpp and small reorg change in online_emission.hpp --- ...mxx_mam_microphysics_process_interface.cpp | 11 + .../src/physics/mam/impl/mam4_amicphys.cpp | 1819 +++++++++++++++++ .../eamxx/src/physics/mam/online_emission.hpp | 26 +- .../src/physics/mam/online_emission_impl.hpp | 52 - 4 files changed, 1850 insertions(+), 58 deletions(-) create mode 100644 components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp delete mode 100644 components/eamxx/src/physics/mam/online_emission_impl.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 57eaf927548..41713b97735 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -70,9 +70,20 @@ void MAMMicrophysics::set_grids( const FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); const FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); +<<<<<<< HEAD using namespace ekat::units; constexpr auto q_unit = kg / kg; // units of mass mixing ratios of tracers constexpr auto n_unit = 1 / kg; // units of number mixing ratios of tracers +======= + config_.amicphys.nucleation = {}; + // config_.amicphys.nucleation.dens_so4a_host = 1770.0; + // config_.amicphys.nucleation.mw_so4a_host = 115.0; + // config_.amicphys.nucleation.newnuc_method_user_choice = 2; + // config_.amicphys.nucleation.pbl_nuc_wang2008_user_choice = 1; + // config_.amicphys.nucleation.adjust_factor_pbl_ratenucl = 1.0; + // config_.amicphys.nucleation.accom_coef_h2so4 = 1.0; + config_.amicphys.nucleation.newnuc_adjust_factor_dnaitdt = 1.0; +>>>>>>> comment out error-inducing code in mam4_amicphys.cpp and small reorg change in online_emission.hpp // -------------------------------------------------------------------------- // These variables are "Required" or pure inputs for the process diff --git a/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp b/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp new file mode 100644 index 00000000000..e241b5ec428 --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp @@ -0,0 +1,1819 @@ +#include +#include +#include +#include +#include +#include + +namespace scream::impl { + +#define MAX_FILENAME_LEN 256 + +using namespace mam4; + +// number of constituents in gas chemistry "work arrays" +KOKKOS_INLINE_FUNCTION +constexpr int gas_pcnst() { + constexpr int gas_pcnst_ = mam4::gas_chemistry::gas_pcnst; + return gas_pcnst_; +} + +// number of aerosol/gas species tendencies +KOKKOS_INLINE_FUNCTION +constexpr int nqtendbb() { return 4; } + +// MAM4 aerosol microphysics configuration data +struct AmicPhysConfig { + // these switches activate various aerosol microphysics processes + bool do_cond; // condensation (a.k.a gas-aerosol exchange) + bool do_rename; // mode "renaming" + bool do_newnuc; // gas -> aerosol nucleation + bool do_coag; // aerosol coagulation + + // configurations for specific aerosol microphysics + mam4::GasAerExchProcess::ProcessConfig condensation; + mam4::NucleationProcess::ProcessConfig nucleation; + + // controls treatment of h2so4 condensation in mam_gasaerexch_1subarea + // 1 = sequential calc. of gas-chem prod then condensation loss + // 2 = simultaneous calc. of gas-chem prod and condensation loss + int gaexch_h2so4_uptake_optaa; + + // controls how nucleation interprets h2so4 concentrations + int newnuc_h2so4_conc_optaa; +}; + +namespace { + +KOKKOS_INLINE_FUNCTION constexpr int nqtendaa() { return 5; } +KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendaa() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendbb() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond() { return 0; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_rnam() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_nnuc() { return 2; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_coag() { return 3; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond_only() { return 4; } +KOKKOS_INLINE_FUNCTION constexpr int iqqcwtend_rnam() { return 0; } +KOKKOS_INLINE_FUNCTION constexpr int maxsubarea() { return 2; } + +// conversion factors +KOKKOS_INLINE_FUNCTION Real fcvt_gas(int gas_id) { + static const Real fcvt_gas_[AeroConfig::num_gas_ids()] = {1, 1, 1}; + return fcvt_gas_[gas_id]; +} +KOKKOS_INLINE_FUNCTION Real fcvt_aer(int aero_id) { + static const Real fcvt_aer_[AeroConfig::num_aerosol_ids()] = {1, 1, 1, 1, 1, 1, 1}; + return fcvt_aer_[aero_id]; +} + +// leave number mix-ratios unchanged (#/kmol-air) +KOKKOS_INLINE_FUNCTION Real fcvt_num() { return 1.0; } +// factor for converting aerosol water mix-ratios from (kg/kg) to (mol/mol) +KOKKOS_INLINE_FUNCTION Real fcvt_wtr() { return 1.0; } + +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_nul() { return 0; } +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_gas() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_aer() { return 2; } +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_num() { return 3; } +KOKKOS_INLINE_FUNCTION int lmapcc_all(int index) { + static const int lmapcc_all_[gas_pcnst()] = { + lmapcc_val_nul(), lmapcc_val_gas(), lmapcc_val_nul(), lmapcc_val_nul(), + lmapcc_val_gas(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_num()}; + return lmapcc_all_[index]; +} + +// Where lmapcc_val_num are defined in lmapcc_all +KOKKOS_INLINE_FUNCTION int numptr_amode(int mode) { + static const int numptr_amode_[AeroConfig::num_modes()] = {12, 17, 25, 29}; + return numptr_amode_[mode]; +} + +// Where lmapcc_val_gas are defined in lmapcc_all +KOKKOS_INLINE_FUNCTION int lmap_gas(int mode) { + static const int lmap_gas_[AeroConfig::num_modes()] = {4, 1}; + return lmap_gas_[mode]; +} +// Where lmapcc_val_aer are defined in lmapcc_all +KOKKOS_INLINE_FUNCTION int lmassptr_amode(int aero_id, int mode) { + static const int lmassptr_amode_[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] = { + {5, 13, 18, 26}, {6, 14, 19, 27}, {7, 15, 20, 28}, {8, 16, 21, -6}, + {9, -6, 22, -6}, {10, -6, 23, -6}, {11, -6, 24, -6}}; + return lmassptr_amode_[aero_id][mode]; +} + +KOKKOS_INLINE_FUNCTION +void subarea_partition_factors( + const Real + q_int_cell_avg, // in grid cell mean interstitial aerosol mixing ratio + const Real + q_cbn_cell_avg, // in grid cell mean cloud-borne aerosol mixing ratio + const Real fcldy, // in cloudy fraction of the grid cell + const Real fclea, // in clear fraction of the grid cell + Real &part_fac_q_int_clea, // out + Real &part_fac_q_int_cldy) // out +{ + // Calculate mixing ratios of each subarea + + // cloud-borne, cloudy subarea + const Real tmp_q_cbn_cldy = q_cbn_cell_avg / fcldy; + // interstitial, cloudy subarea + const Real tmp_q_int_cldy = + haero::max(0.0, ((q_int_cell_avg + q_cbn_cell_avg) - tmp_q_cbn_cldy)); + // interstitial, clear subarea + const Real tmp_q_int_clea = (q_int_cell_avg - fcldy * tmp_q_int_cldy) / fclea; + + // Calculate the corresponding paritioning factors for interstitial aerosols + // using the above-derived subarea mixing ratios plus the constraint that + // the cloud fraction weighted average of subarea mean need to match grid box + // mean. + + // *** question *** + // use same part_fac_q_int_clea/cldy for everything ? + // use one for number and one for all masses (based on total mass) ? + // use separate ones for everything ? + // maybe one for number and one for all masses is best, + // because number and mass have different activation fractions + // *** question *** + + Real tmp_aa = haero::max(1.e-35, tmp_q_int_clea * fclea) / + haero::max(1.e-35, q_int_cell_avg); + tmp_aa = haero::max(0.0, haero::min(1.0, tmp_aa)); + + part_fac_q_int_clea = tmp_aa / fclea; + part_fac_q_int_cldy = (1.0 - tmp_aa) / fcldy; +} + +KOKKOS_INLINE_FUNCTION +void construct_subareas_1gridcell( + const Real cld, // in + const Real relhumgcm, // in + const Real q_pregaschem[gas_pcnst()], // in q TMRs before + // gas-phase chemistry + const Real q_precldchem[gas_pcnst()], // in q TMRs before + // cloud chemistry + const Real qqcw_precldchem[gas_pcnst()], // in qqcw TMRs before + // cloud chemistry + const Real q[gas_pcnst()], // in current tracer mixing ratios (TMRs) + // *** MUST BE #/kmol-air for number + // *** MUST BE mol/mol-air for mass + const Real qqcw[gas_pcnst()], // in like q but for + // cloud-borner tracers + int &nsubarea, // out + int &ncldy_subarea, // out + int &jclea, // out + int &jcldy, // out + bool iscldy_subarea[maxsubarea()], // out + Real afracsub[maxsubarea()], // out + Real relhumsub[maxsubarea()], // out + Real qsub1[gas_pcnst()][maxsubarea()], // out interstitial + Real qsub2[gas_pcnst()][maxsubarea()], // out interstitial + Real qsub3[gas_pcnst()][maxsubarea()], // out interstitial + Real qqcwsub1[gas_pcnst()][maxsubarea()], // out cloud-borne + Real qqcwsub2[gas_pcnst()][maxsubarea()], // out cloud-borne + Real qqcwsub3[gas_pcnst()][maxsubarea()], // outcloud-borne + Real qaerwatsub3[AeroConfig::num_modes()] + [maxsubarea()], // out aerosol water mixing ratios (mol/mol) + Real qaerwat[AeroConfig::num_modes()] // in aerosol water mixing ratio + // (kg/kg, NOT mol/mol) +) { + static constexpr int num_modes = AeroConfig::num_modes(); + // cloud chemistry is only on when cld(i,k) >= 1.0e-5_wp + // it may be that the macrophysics has a higher threshold that this + const Real fcld_locutoff = 1.0e-5; + const Real fcld_hicutoff = 0.999; + + // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios (TMRs, + // mol/mol or #/kmol) + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) + // qgcm1, qgcm2, qgcm3 + // qqcwgcm2, qqcwgcm3 + // qaerwatgcm3 ! aerosol water mixing ratios (mol/mol) + + // -------------------------------------------------------------------------------------- + // Determine the number of sub-areas, their fractional areas, and relative + // humidities + // -------------------------------------------------------------------------------------- + // if cloud fraction ~= 0, the grid-cell has a single clear sub-area + // (nsubarea = 1) if cloud fraction ~= 1, the grid-cell has a single cloudy + // sub-area (nsubarea = 1) otherwise, the grid-cell has a + // clear and a cloudy sub-area (nsubarea = 2) + + Real zfcldy = 0; + nsubarea = 0; + ncldy_subarea = 0; + jclea = 0; + jcldy = 0; + + if (cld < fcld_locutoff) { + nsubarea = 1; + jclea = 1; + } else if (cld > fcld_hicutoff) { + zfcldy = 1.0; + nsubarea = 1; + ncldy_subarea = 1; + jcldy = 1; + } else { + zfcldy = cld; + nsubarea = 2; + ncldy_subarea = 1; + jclea = 1; + jcldy = 2; + } + + const Real zfclea = 1.0 - zfcldy; + for (int i = 0; i < maxsubarea(); ++i) + iscldy_subarea[i] = false; + if (jcldy > 0) + iscldy_subarea[jcldy - 1] = true; + for (int i = 0; i < maxsubarea(); ++i) + afracsub[i] = 0.0; + if (jclea > 0) + afracsub[jclea - 1] = zfclea; + if (jcldy > 0) + afracsub[jcldy - 1] = zfcldy; + + // cldy_rh_sameas_clear is just to match mam_refactor. Compiler should + // optimize away. + const int cldy_rh_sameas_clear = 0; + if (ncldy_subarea <= 0) { + for (int i = 0; i < maxsubarea(); ++i) + relhumsub[i] = relhumgcm; + } else if (cldy_rh_sameas_clear > 0) { + for (int i = 0; i < maxsubarea(); ++i) + relhumsub[i] = relhumgcm; + } else { + if (jcldy > 0) { + relhumsub[jcldy - 1] = 1.0; + if (jclea > 0) { + const Real tmpa = + (relhumgcm - afracsub[jcldy - 1]) / afracsub[jclea - 1]; + relhumsub[jclea - 1] = haero::max(0.0, haero::min(1.0, tmpa)); + } + } + } + + // ---------------------------------------------------------------------------- + // Copy grid cell mean mixing ratios. + // These values, together with cloud fraction and a few assumptions, are used + // in the remainder of the subroutine to calculate the sub-area mean mixing + // ratios. + // ---------------------------------------------------------------------------- + // Interstitial aerosols + Real qgcm1[gas_pcnst()], qgcm2[gas_pcnst()], qgcm3[gas_pcnst()]; + for (int i = 0; i < gas_pcnst(); ++i) { + qgcm1[i] = haero::max(0.0, q_pregaschem[i]); + qgcm2[i] = haero::max(0.0, q_precldchem[i]); + qgcm3[i] = haero::max(0.0, q[i]); + } + + // Cloud-borne aerosols + Real qqcwgcm2[gas_pcnst()], qqcwgcm3[gas_pcnst()]; + for (int i = 0; i < gas_pcnst(); ++i) { + qqcwgcm2[i] = haero::max(0.0, qqcw_precldchem[i]); + qqcwgcm3[i] = haero::max(0.0, qqcw[i]); + } + + // aerosol water + Real qaerwatgcm3[num_modes] = {}; + for (int i = 0; i < num_modes; ++i) { + qaerwatgcm3[i] = haero::max(0.0, qaerwat[i]); + } + + // ---------------------------------------------------------------------------- + // Initialize the subarea mean mixing ratios + // ---------------------------------------------------------------------------- + { + const int n = haero::min(maxsubarea(), nsubarea + 1); + for (int i = 0; i < n; ++i) { + for (int j = 0; j < gas_pcnst(); ++j) { + qsub1[j][i] = 0.0; + qsub2[j][i] = 0.0; + qsub3[j][i] = 0.0; + qqcwsub1[j][i] = 0.0; + qqcwsub2[j][i] = 0.0; + qqcwsub3[j][i] = 0.0; + } + for (int j = 0; j < num_modes; ++j) { + qaerwatsub3[j][i] = 0.0; + } + } + } + + // ************************************************************************************************* + // Calculate initial (i.e., before cond/rnam/nnuc/coag) tracer mixing + // ratios within the sub-areas + // - for all-clear or all-cloudy cases, the sub-area TMRs are equal to the + // grid-cell means + // - for partly cloudy case, they are different. This is primarily + // because the + // interstitial aerosol mixing ratios are assumed lower in the cloudy + // sub-area than in the clear sub-area, because much of the aerosol is + // activated in the cloudy sub-area. + // ************************************************************************************************* + // Category I: partly cloudy case + // ************************************************************************************************* + if ((jclea > 0) && (jcldy > 0) && (jclea + jcldy == 3) && (nsubarea == 2)) { + + // --------------------------------------------------------------------- + // Set GAS mixing ratios in sub-areas (for the condensing gases only!!) + // --------------------------------------------------------------------- + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (lmapcc_all(lmz) == lmapcc_val_gas()) { + + // assume gas in both sub-areas before gas-chem and cloud-chem equal + // grid-cell mean + for (int i = 0; i < nsubarea; ++i) { + qsub1[lmz][i] = qgcm1[lmz]; + qsub2[lmz][i] = qgcm2[lmz]; + } + // assume gas in clear sub-area after cloud-chem equals before + // cloud-chem value + qsub3[lmz][jclea - 1] = qsub2[lmz][jclea - 1]; + // gas in cloud sub-area then determined by grid-cell mean and clear + // values + qsub3[lmz][jcldy - 1] = + (qgcm3[lmz] - zfclea * qsub3[lmz][jclea - 1]) / zfcldy; + + // check that this does not produce a negative value + if (qsub3[lmz][jcldy - 1] < 0.0) { + qsub3[lmz][jcldy - 1] = 0.0; + qsub3[lmz][jclea - 1] = qgcm3[lmz] / zfclea; + } + } + } + // --------------------------------------------------------------------- + // Set CLOUD-BORNE AEROSOL mixing ratios in sub-areas. + // This is straightforward, as the same partitioning factors (0 or 1/f) + // are applied to all mass and number mixing ratios in all modes. + // --------------------------------------------------------------------- + // loop thru log-normal modes + for (int n = 0; n < num_modes; ++n) { + // number - then mass of individual species - of a mode + for (int l2 = -1; l2 < num_species_mode(n); ++l2) { + int lc; + if (l2 == -1) + lc = numptr_amode(n); + else + lc = lmassptr_amode(l2, n); + qqcwsub2[lc][jclea - 1] = 0.0; + qqcwsub2[lc][jcldy - 1] = qqcwgcm2[lc] / zfcldy; + qqcwsub3[lc][jclea - 1] = 0.0; + qqcwsub3[lc][jcldy - 1] = qqcwgcm3[lc] / zfcldy; + } + } + + // --------------------------------------------------------------------- + // Set INTERSTITIAL AEROSOL mixing ratios in sub-areas. + // --------------------------------------------------------------------- + for (int n = 0; n < num_modes; ++n) { + // ------------------------------------- + // Aerosol number + // ------------------------------------- + // grid cell mean, interstitial + Real tmp_q_cellavg_int = qgcm2[numptr_amode(n)]; + // grid cell mean, cloud-borne + Real tmp_q_cellavg_cbn = qqcwgcm2[numptr_amode(n)]; + + Real nmbr_part_fac_clea = 0; + Real nmbr_part_fac_cldy = 0; + subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, + zfclea, nmbr_part_fac_clea, nmbr_part_fac_cldy); + + // Apply the partitioning factors to calculate sub-area mean number + // mixing ratios + + const int la = numptr_amode(n); + + qsub2[la][jclea - 1] = qgcm2[la] * nmbr_part_fac_clea; + qsub2[la][jcldy - 1] = qgcm2[la] * nmbr_part_fac_cldy; + qsub3[la][jclea - 1] = qgcm3[la] * nmbr_part_fac_clea; + qsub3[la][jcldy - 1] = qgcm3[la] * nmbr_part_fac_cldy; + + //------------------------------------- + // Aerosol mass + //------------------------------------- + // For aerosol mass, we use the total grid cell mean + // interstitial/cloud-borne mass mixing ratios to come up with the same + // partitioning for all species in the mode. + + // Compute the total mixing ratios by summing up the individual species + + tmp_q_cellavg_int = 0.0; // grid cell mean, interstitial + tmp_q_cellavg_cbn = 0.0; // grid cell mean, cloud-borne + + for (int l2 = 0; l2 < num_species_mode(n); ++l2) { + tmp_q_cellavg_int += qgcm2[lmassptr_amode(l2, n)]; + tmp_q_cellavg_cbn += qqcwgcm2[lmassptr_amode(l2, n)]; + } + Real mass_part_fac_clea = 0; + Real mass_part_fac_cldy = 0; + // Calculate the partitioning factors + subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, + zfclea, mass_part_fac_clea, mass_part_fac_cldy); + + // Apply the partitioning factors to calculate sub-area mean mass mixing + // ratios + + for (int l2 = 0; l2 < num_species_mode(n); ++l2) { + const int la = lmassptr_amode(l2, n); + + qsub2[la][jclea - 1] = qgcm2[la] * mass_part_fac_clea; + qsub2[la][jcldy - 1] = qgcm2[la] * mass_part_fac_cldy; + qsub3[la][jclea - 1] = qgcm3[la] * mass_part_fac_clea; + qsub3[la][jcldy - 1] = qgcm3[la] * mass_part_fac_cldy; + } + } + + // ************************************************************************************************* + // Category II: all clear, or cld < 1e-5 + // In this case, zfclea=1 and zfcldy=0 + // ************************************************************************************************* + } else if ((jclea == 1) && (jcldy == 0) && (nsubarea == 1)) { + // + // put all the gases and interstitial aerosols in the clear sub-area + // and set mix-ratios = 0 in cloudy sub-area + // for cloud-borne aerosol, do nothing + // because the grid-cell-mean cloud-borne aerosol will be left + // unchanged (i.e., this routine only changes qqcw when cld >= 1e-5) + // + + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (0 < lmapcc_all(lmz)) { + qsub1[lmz][jclea - 1] = qgcm1[lmz]; + qsub2[lmz][jclea - 1] = qgcm2[lmz]; + qsub3[lmz][jclea - 1] = qgcm3[lmz]; + qqcwsub2[lmz][jclea - 1] = qqcwgcm2[lmz]; + qqcwsub3[lmz][jclea - 1] = qqcwgcm3[lmz]; + } + } + // ************************************************************************************************* + // Category III: all cloudy, or cld > 0.999 + // in this case, zfcldy= and zfclea=0 + // ************************************************************************************************* + } else if ((jclea == 0) && (jcldy == 1) && (nsubarea == 1)) { + // + // put all the gases and interstitial aerosols in the cloudy sub-area + // and set mix-ratios = 0 in clear sub-area + // + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (0 < lmapcc_all(lmz)) { + qsub1[lmz][jcldy - 1] = qgcm1[lmz]; + qsub2[lmz][jcldy - 1] = qgcm2[lmz]; + qsub3[lmz][jcldy - 1] = qgcm3[lmz]; + qqcwsub2[lmz][jcldy - 1] = qqcwgcm2[lmz]; + qqcwsub3[lmz][jcldy - 1] = qqcwgcm3[lmz]; + } + } + // ************************************************************************************************* + } else { // this should not happen + EKAT_KERNEL_REQUIRE_MSG(false, "*** modal_aero_amicphys - bad jclea, jcldy, nsubarea!"); + } + // ************************************************************************************************* + + // ------------------------------------------------------------------------------------ + // aerosol water -- how to treat this in sub-areas needs more work/thinking + // currently modal_aero_water_uptake calculates qaerwat using + // the grid-cell mean interstital-aerosol mix-rats and the clear-area rh + for (int jsub = 0; jsub < nsubarea; ++jsub) + for (int i = 0; i < num_modes; ++i) + qaerwatsub3[i][jsub] = qaerwatgcm3[i]; + + // ------------------------------------------------------------------------------------ + if (nsubarea == 1) { + // the j=1 subarea is used for some diagnostics + // but is not used in actual calculations + const int j = 1; + for (int i = 0; i < gas_pcnst(); ++i) { + qsub1[i][j] = 0.0; + qsub2[i][j] = 0.0; + qsub3[i][j] = 0.0; + qqcwsub2[i][j] = 0.0; + qqcwsub3[i][j] = 0.0; + } + } +} + +KOKKOS_INLINE_FUNCTION +void mam_amicphys_1subarea_clear( + const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, + const int nsubarea, const bool iscldy_subarea, const Real afracsub, + const Real temp, const Real pmid, const Real pdel, const Real zmid, + const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], + Real dgn_awet[AeroConfig::num_modes()], + Real wetdens[AeroConfig::num_modes()], + const Real qgas1[AeroConfig::num_gas_ids()], + const Real qgas3[AeroConfig::num_gas_ids()], + Real qgas4[AeroConfig::num_gas_ids()], + Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], + const Real qnum3[AeroConfig::num_modes()], + Real qnum4[AeroConfig::num_modes()], + Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], + const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] + [nqtendaa()], + const Real qwtr3[AeroConfig::num_modes()], + Real qwtr4[AeroConfig::num_modes()]) { + static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); + static constexpr int num_modes = AeroConfig::num_modes(); + static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); + + static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); + // Turn off nh3 for now. This is a future enhancement. + static constexpr int igas_nh3 = -999888777; // Same as mam_refactor + static constexpr int iaer_so4 = static_cast(AeroId::SO4); + static constexpr int iaer_pom = static_cast(AeroId::POM); + + const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, + AeroId::None}; + + const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { + {true, true, true, true}, + {true, true, true, true}, + {false, false, false, false}}; + enum { NA, ANAL, IMPL }; + const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; + + // air molar density (kmol/m3) + // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] + const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor + const Real aircon = pmid / (1000 * r_universal * temp); + const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, + 0.58778666490211906, 0.47000362924573563}; + const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; + // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) + // qgas3, qaer3, qnum3 are the current incoming TMRs + // qgas4, qaer4, qnum4 are the updated outgoing TMRs + // + // this routine calculates changes involving + // gas-aerosol exchange (condensation/evaporation) + // growth from smaller to larger modes (renaming) due to condensation + // new particle nucleation + // coagulation + // transfer of particles from hydrophobic modes to hydrophilic modes + // (aging) + // due to condensation and coagulation + // + // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios + // XXX=gas - gas species + // XXX=aer - aerosol mass species (excluding water) + // XXX=wat - aerosol water + // XXX=num - aerosol number + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - current incoming values (before gas-aerosol exchange, newnuc, + // coag) N=4 - updated outgoing values (after gas-aerosol exchange, + // newnuc, coag) + // + // qXXX_delaa are TMR changes (not tendencies) + // for different processes, which are used to produce history output + // for a clear sub-area, the processes are condensation/evaporation (and + // associated aging), renaming, coagulation, and nucleation + + Real qgas_cur[num_gas_ids]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_cur[i] = qgas3[i]; + Real qaer_cur[num_aerosol_ids][num_modes]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_cur[i][j] = qaer3[i][j]; + + Real qnum_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qnum_cur[j] = qnum3[j]; + Real qwtr_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qwtr_cur[j] = qwtr3[j]; + + // qgas_netprod_otrproc = gas net production rate from other processes + // such as gas-phase chemistry and emissions (mol/mol/s) + // this allows the condensation (gasaerexch) routine to apply production and + // condensation loss + // together, which is more accurate numerically + // NOTE - must be >= zero, as numerical method can fail when it is negative + // NOTE - currently only the values for h2so4 and nh3 should be non-zero + Real qgas_netprod_otrproc[num_gas_ids] = {}; + if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { + for (int igas = 0; igas < num_gas_ids; ++igas) { + if (igas == igas_h2so4 || igas == igas_nh3) { + // if config.gaexch_h2so4_uptake_optaa == 2, then + // if qgas increases from pre-gaschem to post-cldchem, + // start from the pre-gaschem mix-ratio and add in the production + // during the integration + // if it decreases, + // start from post-cldchem mix-ratio + // *** currently just do this for h2so4 and nh3 + qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; + if (qgas_netprod_otrproc[igas] >= 0.0) + qgas_cur[igas] = qgas1[igas]; + else + qgas_netprod_otrproc[igas] = 0.0; + } + } + } + Real qgas_del_cond[num_gas_ids] = {}; + Real qgas_del_nnuc[num_gas_ids] = {}; + Real qgas_del_cond_only[num_gas_ids] = {}; + Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; + Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; + Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; + Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; + Real qnum_del_cond[num_modes] = {}; + Real qnum_del_rnam[num_modes] = {}; + Real qnum_del_nnuc[num_modes] = {}; + Real qnum_del_coag[num_modes] = {}; + Real qnum_delsub_cond[num_modes] = {}; + Real qnum_delsub_coag[num_modes] = {}; + Real qnum_del_cond_only[num_modes] = {}; + Real dnclusterdt = 0.0; + + const int ntsubstep = 1; + Real dtsubstep = deltat; + if (ntsubstep > 1) + dtsubstep = deltat / ntsubstep; + Real del_h2so4_gasprod = + haero::max(qgas3[igas_h2so4] - qgas1[igas_h2so4], 0.0) / ntsubstep; + + // loop over multiple time sub-steps + for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { + // gas-aerosol exchange + Real uptkrate_h2so4 = 0.0; + Real del_h2so4_aeruptk = 0.0; + Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + Real qgas_avg[num_gas_ids] = {}; + Real qnum_sv1[num_modes] = {}; + Real qaer_sv1[num_aerosol_ids][num_modes] = {}; + Real qgas_sv1[num_gas_ids] = {}; + + if (config.do_cond) { + + const bool l_calc_gas_uptake_coeff = jtsubstep == 1; + Real uptkaer[num_gas_ids][num_modes] = {}; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_sv1[i] = qgas_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + + // time sub-step + const Real dtsub_soa_fixed = -1.0; + // Integration order + const int nghq = 2; + const int ntot_soamode = 4; + int niter_out = 0; + Real g0_soa_out = 0; + // gasaerexch::mam_gasaerexch_1subarea( + // nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, + // iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, + // eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, + // aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, + // qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, + // uptkrate_h2so4, niter_out, g0_soa_out); + + if (config.newnuc_h2so4_conc_optaa == 11) + qgas_avg[igas_h2so4] = + 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); + else if (config.newnuc_h2so4_conc_optaa == 12) + qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond[i] += + (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); + + for (int i = 0; i < num_modes; ++i) + qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + + // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation + // calculations + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond_only[i] = qgas_del_cond[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; + for (int i = 0; i < num_modes; ++i) + qnum_del_cond_only[i] = qnum_delsub_cond[i]; + del_h2so4_aeruptk = + qgas_cur[igas_h2so4] - + (qgas_sv1[igas_h2so4] + qgas_netprod_otrproc[igas_h2so4] * dtsubstep); + } else { + for (int i = 0; i < num_gas_ids; ++i) + qgas_avg[i] = qgas_cur[i]; + } + + // renaming after "continuous growth" + if (config.do_rename) { + constexpr int nmodes = AeroConfig::num_modes(); + constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); + const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT + const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; + + Real qnumcw_cur[num_modes] = {}; + Real qaercw_cur[num_aerosol_ids][num_modes] = {}; + Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + Real mean_std_dev[nmodes]; + Real fmode_dist_tail_fac[nmodes]; + Real v2n_lo_rlx[nmodes]; + Real v2n_hi_rlx[nmodes]; + Real ln_diameter_tail_fac[nmodes]; + int num_pairs = 0; + Real diameter_cutoff[nmodes]; + Real ln_dia_cutoff[nmodes]; + Real diameter_threshold[nmodes]; + Real mass_2_vol[naerosol_species] = {0.15, + 6.4971751412429377e-002, + 0.15, + 7.0588235294117650e-003, + 3.0789473684210526e-002, + 5.1923076923076926e-002, + 156.20986883198000}; + + rename::find_renaming_pairs(dest_mode_of_mode, // in + mean_std_dev, // out + fmode_dist_tail_fac, // out + v2n_lo_rlx, // out + v2n_hi_rlx, // out + ln_diameter_tail_fac, // out + num_pairs, // out + diameter_cutoff, // out + ln_dia_cutoff, diameter_threshold); + + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + Real dgnum_amode[nmodes]; + for (int m = 0; m < nmodes; ++m) { + dgnum_amode[m] = modes(m).nom_diameter; + } + + { + Real qmol_i_cur[num_modes][num_aerosol_ids]; + Real qmol_i_del[num_modes][num_aerosol_ids]; + Real qmol_c_cur[num_modes][num_aerosol_ids]; + Real qmol_c_del[num_modes][num_aerosol_ids]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qmol_i_cur[i][j] = qaer_cur[j][i]; + qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; + qmol_c_cur[i][j] = qaercw_cur[j][i]; + qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; + } + Rename rename; + rename.mam_rename_1subarea_( + iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, + mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, + ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, + diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, + qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); + + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qaer_cur[j][i] = qmol_i_cur[i][j]; + qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; + qaercw_cur[j][i] = qmol_c_cur[i][j]; + qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; + } + } + + for (int i = 0; i < num_modes; ++i) + qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; + } + + // new particle formation (nucleation) + if (config.do_newnuc) { + for (int i = 0; i < num_gas_ids; ++i) + qgas_sv1[i] = qgas_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + Real qaer_cur_tmp[num_modes][num_aerosol_ids]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qaer_sv1[j][i] = qaer_cur[j][i]; + qaer_cur_tmp[i][j] = qaer_cur[j][i]; + } + Real dnclusterdt_substep = 0; + Real dndt_ait = 0; + Real dmdt_ait = 0; + Real dso4dt_ait = 0; + Real dnh4dt_ait = 0; + Nucleation nucleation; + Nucleation::Config config; + // config.dens_so4a_host = 1770; + config.mw_nh4a_host = 115; + config.mw_so4a_host = 115; + // config.accom_coef_h2so4 = 0.65; + AeroConfig aero_config; + nucleation.init(aero_config, config); + // new version after switching from box model version of nucleation + // compute_tendencies_( + // // Real deltat, + // // Real temp_in, + // // Real press_in, + // // Real zm_in, + // // Real pblh_in, + // // Real relhum, + // // Real uptkrate_h2so4, + // const int nsize, // missing? + // const Real dp_lo_mode, + // const Real dp_hi_mode, + // // const Real qgas_cur[num_gases], + // // const Real qgas_avg[num_gases], + // int &isize_nuc, + // // Real &qnuma_del, + // // Real &qso4a_del, + // // Real &qnh4a_del, + // // Real &qh2so4_del, + // Real &qnh3_del, + // // Real &dnclusterdt + // ) + // old, box-model-based version + // nucleation.compute_tendencies_( + // dtsubstep, temp, pmid, aircon, zmid, pblh, relhum, uptkrate_h2so4, + // del_h2so4_gasprod, del_h2so4_aeruptk, qgas_cur, qgas_avg, qnum_cur, + // qaer_cur_tmp, qwtr_cur, dndt_ait, dmdt_ait, dso4dt_ait, dnh4dt_ait, + // dnclusterdt_substep); + // nucleation.compute_tendencies_( + // dtsubstep, // deltat + // temp, // temp_in + // pmid, // press_in + // aircon, // ? + // zmid, // zm_in + // pblh, // pblh_in + // relhum, // relhum + // uptkrate_h2so4, // uptkrate_h2so4 + // // nsize? + // // dp_lo_mode? + // // dp_hi_mode? + // del_h2so4_aeruptk, // uptkrate_h2so4? + // qgas_cur, // qgas_cur[num_gases]? + // qgas_avg, // qgas_avg[num_gases]? + // // isize_nuc? + // qnum_cur, // qnuma_del? + // qaer_cur_tmp, // ? + // qwtr_cur, // ? + // dndt_ait, // + // dmdt_ait, // + // dso4dt_ait, // qso4a_del + // dnh4dt_ait, // qnh4a_del + // del_h2so4_gasprod, // qh2so4_del? + // // qnh3_del? + // dnclusterdt_substep // dnclusterdt + // ); + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_cur[j][i] = qaer_cur_tmp[i][j]; + + //! Apply the tendencies to the prognostics. + const int nait = static_cast(ModeIndex::Aitken); + qnum_cur[nait] += dndt_ait * dtsubstep; + + if (dso4dt_ait > 0.0) { + static constexpr int iaer_so4 = static_cast(AeroId::SO4); + static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); + + Real delta_q = dso4dt_ait * dtsubstep; + qaer_cur[iaer_so4][nait] += delta_q; + delta_q = haero::min(delta_q, qgas_cur[igas_h2so4]); + qgas_cur[igas_h2so4] -= delta_q; + } + + if (igas_nh3 > 0 && dnh4dt_ait > 0.0) { + static constexpr int iaer_nh4 = + -9999999; // static_cast(AeroId::NH4); + + Real delta_q = dnh4dt_ait * dtsubstep; + qaer_cur[iaer_nh4][nait] += delta_q; + delta_q = haero::min(delta_q, qgas_cur[igas_nh3]); + qgas_cur[igas_nh3] -= delta_q; + } + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_nnuc[i] += (qgas_cur[i] - qgas_sv1[i]); + for (int i = 0; i < num_modes; ++i) + qnum_del_nnuc[i] += (qnum_cur[i] - qnum_sv1[i]); + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_nnuc[j][i] += (qaer_cur[j][i] - qaer_sv1[j][i]); + + dnclusterdt = dnclusterdt + dnclusterdt_substep * (dtsubstep / deltat); + } + + // coagulation part + if (config.do_coag) { + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + // coagulation::mam_coag_1subarea(dtsubstep, temp, pmid, aircon, dgn_a, + // dgn_awet, wetdens, qnum_cur, qaer_cur, + // qaer_delsub_coag_in); + for (int i = 0; i < num_modes; ++i) + qnum_delsub_coag[i] = qnum_cur[i] - qnum_sv1[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_delsub_coag[j][i] = qaer_cur[j][i] - qaer_sv1[j][i]; + } + + // primary carbon aging + + aging::mam_pcarbon_aging_1subarea( + dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, + qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); + + // accumulate sub-step q-dels + if (config.do_coag) { + for (int i = 0; i < num_modes; ++i) + qnum_del_coag[i] += qnum_delsub_coag[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_coag[j][i] += qaer_delsub_coag[j][i]; + } + if (config.do_cond) { + for (int i = 0; i < num_modes; ++i) + qnum_del_cond[i] += qnum_delsub_cond[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; + } + } + + // final mix ratios + for (int i = 0; i < num_gas_ids; ++i) + qgas4[i] = qgas_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer4[j][i] = qaer_cur[j][i]; + for (int i = 0; i < num_modes; ++i) + qnum4[i] = qnum_cur[i]; + for (int i = 0; i < num_modes; ++i) + qwtr4[i] = qwtr_cur[i]; + + // final mix ratio changes + for (int i = 0; i < num_gas_ids; ++i) { + qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; + qgas_delaa[i][iqtend_rnam()] = 0.0; + qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; + qgas_delaa[i][iqtend_coag()] = 0.0; + qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; + } + for (int i = 0; i < num_modes; ++i) { + qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; + qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; + qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; + qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; + qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; + } + for (int j = 0; j < num_aerosol_ids; ++j) { + for (int i = 0; i < num_modes; ++i) { + qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; + qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; + qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; + qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; + qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; + } + } +} + +KOKKOS_INLINE_FUNCTION +void mam_amicphys_1subarea_cloudy( + const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, + const int nsubarea, const bool iscldy_subarea, const Real afracsub, + const Real temp, const Real pmid, const Real pdel, const Real zmid, + const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], + Real dgn_awet[AeroConfig::num_modes()], + Real wetdens[AeroConfig::num_modes()], + const Real qgas1[AeroConfig::num_gas_ids()], + const Real qgas3[AeroConfig::num_gas_ids()], + Real qgas4[AeroConfig::num_gas_ids()], + Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], + const Real qnum3[AeroConfig::num_modes()], + Real qnum4[AeroConfig::num_modes()], + Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], + const Real qaer2[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] + [nqtendaa()], + const Real qwtr3[AeroConfig::num_modes()], + Real qwtr4[AeroConfig::num_modes()], + const Real qnumcw3[AeroConfig::num_modes()], + Real qnumcw4[AeroConfig::num_modes()], + Real qnumcw_delaa[AeroConfig::num_modes()][nqqcwtendaa()], + const Real qaercw2[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], + const Real qaercw3[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], + Real qaercw4[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], + Real qaercw_delaa[AeroConfig::num_gas_ids()][AeroConfig::num_modes()] + [nqqcwtendaa()]) { + + // + // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) + // qgas3, qaer3, qaercw3, qnum3, qnumcw3 are the current incoming TMRs + // qgas4, qaer4, qaercw4, qnum4, qnumcw4 are the updated outgoing TMRs + // + // when config.do_cond = false, this routine only calculates changes involving + // growth from smaller to larger modes (renaming) following cloud chemistry + // so gas TMRs are not changed + // when config.do_cond = true, this routine also calculates changes involving + // gas-aerosol exchange (condensation/evaporation) + // transfer of particles from hydrophobic modes to hydrophilic modes + // (aging) + // due to condensation + // currently this routine does not do + // new particle nucleation - because h2so4 gas conc. should be very low in + // cloudy air coagulation - because cloud-borne aerosol would need to be + // included + // + + // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios + // XXX=gas - gas species + // XXX=aer - aerosol mass species (excluding water) + // XXX=wat - aerosol water + // XXX=num - aerosol number + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - current incoming values (before gas-aerosol exchange, newnuc, + // coag) N=4 - updated outgoing values (after gas-aerosol exchange, + // newnuc, coag) + // + // qXXX_delaa are TMR changes (not tendencies) + // for different processes, which are used to produce history output + // for a clear sub-area, the processes are condensation/evaporation (and + // associated aging), + // renaming, coagulation, and nucleation + + // qxxx_del_yyyy are mix-ratio changes over full time step (deltat) + // qxxx_delsub_yyyy are mix-ratio changes over time sub-step (dtsubstep) + + static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); + static constexpr int num_modes = AeroConfig::num_modes(); + static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); + + static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); + // Turn off nh3 for now. This is a future enhancement. + static constexpr int igas_nh3 = -999888777; // Same as mam_refactor + static constexpr int iaer_so4 = static_cast(AeroId::SO4); + static constexpr int iaer_pom = static_cast(AeroId::POM); + + const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, + AeroId::None}; + const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { + {true, true, true, true}, + {true, true, true, true}, + {false, false, false, false}}; + enum { NA, ANAL, IMPL }; + const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; + // air molar density (kmol/m3) + // In order to try to match the results in mam_refactor + // set r_universal as [mJ/(mol)] as in mam_refactor. + // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] + const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor + const Real aircon = pmid / (1000 * r_universal * temp); + const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, + 0.58778666490211906, 0.47000362924573563}; + const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; + + Real qgas_cur[num_gas_ids]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_cur[i] = qgas3[i]; + Real qaer_cur[num_aerosol_ids][num_modes]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_cur[i][j] = qaer3[i][j]; + + Real qnum_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qnum_cur[j] = qnum3[j]; + Real qwtr_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qwtr_cur[j] = qwtr3[j]; + + Real qnumcw_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qnumcw_cur[j] = qnumcw3[j]; + + Real qaercw_cur[num_gas_ids][num_modes]; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_cur[i][j] = qaercw3[i][j]; + + Real qgas_netprod_otrproc[num_gas_ids] = {}; + if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { + for (int igas = 0; igas < num_gas_ids; ++igas) { + if (igas == igas_h2so4 || igas == igas_nh3) { + // if gaexch_h2so4_uptake_optaa == 2, then + // if qgas increases from pre-gaschem to post-cldchem, + // start from the pre-gaschem mix-ratio and add in the production + // during the integration + // if it decreases, + // start from post-cldchem mix-ratio + // *** currently just do this for h2so4 and nh3 + qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; + if (qgas_netprod_otrproc[igas] >= 0.0) + qgas_cur[igas] = qgas1[igas]; + else + qgas_netprod_otrproc[igas] = 0.0; + } + } + } + Real qgas_del_cond[num_gas_ids] = {}; + Real qgas_del_nnuc[num_gas_ids] = {}; + Real qgas_del_cond_only[num_gas_ids] = {}; + Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; + Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; + Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; + Real qaercw_del_rnam[num_aerosol_ids][num_modes] = {}; + Real qnum_del_cond[num_modes] = {}; + Real qnum_del_rnam[num_modes] = {}; + Real qnum_del_nnuc[num_modes] = {}; + Real qnum_del_coag[num_modes] = {}; + Real qnum_delsub_cond[num_modes] = {}; + Real qnum_delsub_coag[num_modes] = {}; + Real qnum_del_cond_only[num_modes] = {}; + Real qnumcw_del_rnam[num_modes] = {}; + Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; + + const int ntsubstep = 1; + Real dtsubstep = deltat; + if (ntsubstep > 1) + dtsubstep = deltat / ntsubstep; + + // loop over multiple time sub-steps + + for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { + // gas-aerosol exchange + Real uptkrate_h2so4 = 0.0; + Real qgas_avg[num_gas_ids] = {}; + Real qgas_sv1[num_gas_ids] = {}; + Real qnum_sv1[num_modes] = {}; + Real qaer_sv1[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + + if (config.do_cond) { + + const bool l_calc_gas_uptake_coeff = jtsubstep == 1; + Real uptkaer[num_gas_ids][num_modes] = {}; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_sv1[i] = qgas_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + + const int nghq = 2; + const int ntot_soamode = 4; + int niter_out = 0; + Real g0_soa_out = 0; + // time sub-step + const Real dtsub_soa_fixed = -1.0; + // gasaerexch::mam_gasaerexch_1subarea( + // nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, + // iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, + // eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, + // aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, + // qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, + // uptkrate_h2so4, niter_out, g0_soa_out); + + if (config.newnuc_h2so4_conc_optaa == 11) + qgas_avg[igas_h2so4] = + 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); + else if (config.newnuc_h2so4_conc_optaa == 12) + qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond[i] += + (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); + + for (int i = 0; i < num_modes; ++i) + qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + + // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation + // calculations + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond_only[i] = qgas_del_cond[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; + for (int i = 0; i < num_modes; ++i) + qnum_del_cond_only[i] = qnum_delsub_cond[i]; + + } else { + for (int i = 0; i < num_gas_ids; ++i) + qgas_avg[i] = qgas_cur[i]; + } + // renaming after "continuous growth" + if (config.do_rename) { + constexpr int nmodes = AeroConfig::num_modes(); + constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); + const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT + const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; + + Real qnumcw_cur[num_modes] = {}; + Real qaercw_cur[num_aerosol_ids][num_modes] = {}; + Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + Real mean_std_dev[nmodes]; + Real fmode_dist_tail_fac[nmodes]; + Real v2n_lo_rlx[nmodes]; + Real v2n_hi_rlx[nmodes]; + Real ln_diameter_tail_fac[nmodes]; + int num_pairs = 0; + Real diameter_cutoff[nmodes]; + Real ln_dia_cutoff[nmodes]; + Real diameter_threshold[nmodes]; + Real mass_2_vol[naerosol_species] = {0.15, + 6.4971751412429377e-002, + 0.15, + 7.0588235294117650e-003, + 3.0789473684210526e-002, + 5.1923076923076926e-002, + 156.20986883198000}; + + rename::find_renaming_pairs(dest_mode_of_mode, // in + mean_std_dev, // out + fmode_dist_tail_fac, // out + v2n_lo_rlx, // out + v2n_hi_rlx, // out + ln_diameter_tail_fac, // out + num_pairs, // out + diameter_cutoff, // out + ln_dia_cutoff, diameter_threshold); + + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + Real dgnum_amode[nmodes]; + for (int m = 0; m < nmodes; ++m) { + dgnum_amode[m] = modes(m).nom_diameter; + } + + // qaercw_delsub_grow4rnam = change in qaercw from cloud chemistry + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_delsub_grow4rnam[i][j] = + (qaercw3[i][j] - qaercw2[i][j]) / ntsubstep; + Real qnumcw_sv1[num_modes]; + for (int i = 0; i < num_modes; ++i) + qnumcw_sv1[i] = qnumcw_cur[i]; + Real qaercw_sv1[num_aerosol_ids][num_modes]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_sv1[i][j] = qaercw_cur[i][j]; + + { + Real qmol_i_cur[num_modes][num_aerosol_ids]; + Real qmol_i_del[num_modes][num_aerosol_ids]; + Real qmol_c_cur[num_modes][num_aerosol_ids]; + Real qmol_c_del[num_modes][num_aerosol_ids]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qmol_i_cur[i][j] = qaer_cur[j][i]; + qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; + qmol_c_cur[i][j] = qaercw_cur[j][i]; + qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; + } + + Rename rename; + rename.mam_rename_1subarea_( + iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, + mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, + ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, + diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, + qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); + + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qaer_cur[j][i] = qmol_i_cur[i][j]; + qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; + qaercw_cur[j][i] = qmol_c_cur[i][j]; + qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; + } + } + for (int i = 0; i < num_modes; ++i) + qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; + for (int i = 0; i < num_modes; ++i) + qnumcw_del_rnam[i] += qnumcw_cur[i] - qnumcw_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_del_rnam[i][j] += qaercw_cur[i][j] - qaercw_sv1[i][j]; + } + + // primary carbon aging + if (config.do_cond) { + aging::mam_pcarbon_aging_1subarea( + dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, + qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); + } + // accumulate sub-step q-dels + if (config.do_cond) { + for (int i = 0; i < num_modes; ++i) + qnum_del_cond[i] += qnum_delsub_cond[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; + } + } + // final mix ratios + for (int i = 0; i < num_gas_ids; ++i) + qgas4[i] = qgas_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer4[j][i] = qaer_cur[j][i]; + for (int i = 0; i < num_modes; ++i) + qnum4[i] = qnum_cur[i]; + for (int i = 0; i < num_modes; ++i) + qwtr4[i] = qwtr_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnumcw4[i] = qnumcw_cur[i]; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw4[i][j] = qaercw_cur[i][j]; + + // final mix ratio changes + for (int i = 0; i < num_gas_ids; ++i) { + qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; + qgas_delaa[i][iqtend_rnam()] = 0.0; + qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; + qgas_delaa[i][iqtend_coag()] = 0.0; + qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; + } + for (int i = 0; i < num_modes; ++i) { + qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; + qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; + qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; + qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; + qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; + } + for (int j = 0; j < num_aerosol_ids; ++j) { + for (int i = 0; i < num_modes; ++i) { + qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; + qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; + qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; + qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; + qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; + } + } + for (int i = 0; i < num_modes; ++i) + qnumcw_delaa[i][iqqcwtend_rnam()] = qnumcw_del_rnam[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_delaa[i][j][iqqcwtend_rnam()] = qaercw_del_rnam[i][j]; +} + +KOKKOS_INLINE_FUNCTION +void mam_amicphys_1gridcell( + const AmicPhysConfig& config, const int nstep, const Real deltat, const int nsubarea, + const int ncldy_subarea, const bool iscldy_subarea[maxsubarea()], + const Real afracsub[maxsubarea()], const Real temp, const Real pmid, + const Real pdel, const Real zmid, const Real pblh, + const Real relhumsub[maxsubarea()], Real dgn_a[AeroConfig::num_modes()], + Real dgn_awet[AeroConfig::num_modes()], + Real wetdens[AeroConfig::num_modes()], + const Real qsub1[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qsub2[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qqcwsub2[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qsub3[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qqcwsub3[AeroConfig::num_gas_ids()][maxsubarea()], + Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()], + Real qsub4[AeroConfig::num_gas_ids()][maxsubarea()], + Real qqcwsub4[AeroConfig::num_gas_ids()][maxsubarea()], + Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()], + Real qsub_tendaa[AeroConfig::num_gas_ids()][nqtendaa()][maxsubarea()], + Real qqcwsub_tendaa[AeroConfig::num_gas_ids()][nqqcwtendaa()][maxsubarea()]) { + + // + // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) + // qsub3 and qqcwsub3 are the incoming current TMRs + // qsub4 and qqcwsub4 are the outgoing updated TMRs + // + // qsubN and qqcwsubN (N=1:4) are tracer mixing ratios (TMRs, mol/mol or + // #/kmol) in sub-areas + // currently there are just clear and cloudy sub-areas + // the N=1:4 have same meanings as for qgcmN + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) + // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) + // qsub_tendaa and qqcwsub_tendaa are TMR tendencies + // for different processes, which are used to produce history output + // the processes are condensation/evaporation (and associated aging), + // renaming, coagulation, and nucleation + + static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); + static constexpr int num_modes = AeroConfig::num_modes(); + static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); + + // the q--4 values will be equal to q--3 values unless they get changed + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < maxsubarea(); ++j) { + qsub4[i][j] = qsub3[i][j]; + qqcwsub4[i][j] = qqcwsub3[i][j]; + } + for (int i = 0; i < num_modes; ++i) + for (int j = 0; j < maxsubarea(); ++j) + qaerwatsub4[i][j] = qaerwatsub3[i][j]; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < nqtendaa(); ++j) + for (int k = 0; k < maxsubarea(); ++k) + qsub_tendaa[i][j][k] = 0; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + for (int k = 0; k < maxsubarea(); ++k) + qqcwsub_tendaa[i][j][k] = 0.0; + + for (int jsub = 0; jsub < nsubarea; ++jsub) { + AmicPhysConfig sub_config = config; + if (iscldy_subarea[jsub]) { + sub_config.do_cond = config.do_cond; + sub_config.do_rename = config.do_rename; + sub_config.do_newnuc = false; + sub_config.do_coag = false; + } + const bool do_map_gas_sub = sub_config.do_cond || sub_config.do_newnuc; + + // map incoming sub-area mix-ratios to gas/aer/num arrays + Real qgas1[num_gas_ids] = {}; + Real qgas3[num_gas_ids] = {}; + Real qgas4[num_gas_ids] = {}; + if (do_map_gas_sub) { + // for cldy subarea, only do gases if doing gaexch + for (int igas = 0; igas < 2; ++igas) { + const int l = lmap_gas(igas); + qgas1[igas] = qsub1[l][jsub] * fcvt_gas(igas); + qgas3[igas] = qsub3[l][jsub] * fcvt_gas(igas); + qgas4[igas] = qgas3[igas]; + } + } + Real qaer2[num_aerosol_ids][num_modes] = {}; + Real qaer3[num_aerosol_ids][num_modes] = {}; + Real qnum3[num_modes] = {}; + Real qaer4[num_aerosol_ids][num_modes] = {}; + Real qnum4[num_modes] = {}; + Real qwtr3[num_modes] = {}; + Real qwtr4[num_modes] = {}; + for (int n = 0; n < num_modes; ++n) { + qnum3[n] = qsub3[n][jsub] * fcvt_num(); + qnum4[n] = qnum3[n]; + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qaer2[iaer][n] = qsub2[iaer][jsub] * fcvt_aer(iaer); + qaer3[iaer][n] = qsub3[iaer][jsub] * fcvt_aer(iaer); + qaer4[iaer][n] = qaer3[iaer][n]; + } + qwtr3[n] = qaerwatsub3[n][jsub] * fcvt_wtr(); + qwtr4[n] = qwtr3[n]; + } + Real qaercw2[num_aerosol_ids][num_modes] = {}; + Real qaercw3[num_aerosol_ids][num_modes] = {}; + Real qnumcw3[num_modes] = {}; + Real qaercw4[num_aerosol_ids][num_modes] = {}; + Real qnumcw4[num_modes] = {}; + if (iscldy_subarea[jsub]) { + // only do cloud-borne for cloudy + for (int n = 0; n < num_modes; ++n) { + qnumcw3[n] = qqcwsub3[n][jsub] * fcvt_num(); + qnumcw4[n] = qnumcw3[n]; + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qaercw2[iaer][n] = qqcwsub2[n][jsub] * fcvt_aer(iaer); + qaercw3[iaer][n] = qqcwsub3[n][jsub] * fcvt_aer(iaer); + qaercw4[iaer][n] = qaercw3[iaer][n]; + } + } + } + + Real qgas_delaa[num_gas_ids][nqtendaa()] = {}; + Real qnum_delaa[num_modes][nqtendaa()] = {}; + Real qnumcw_delaa[num_modes][nqqcwtendaa()] = {}; + Real qaer_delaa[num_aerosol_ids][num_modes][nqtendaa()] = {}; + Real qaercw_delaa[num_aerosol_ids][num_modes][nqqcwtendaa()] = {}; + + if (iscldy_subarea[jsub]) { + mam_amicphys_1subarea_cloudy(sub_config, nstep, deltat, + jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, + pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, + qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer2, qaer3, + qaer4, qaer_delaa, qwtr3, qwtr4, qnumcw3, qnumcw4, qnumcw_delaa, + qaercw2, qaercw3, qaercw4, qaercw_delaa); + } else { + mam_amicphys_1subarea_clear(sub_config, nstep, deltat, + jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, + pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, + qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer3, qaer4, + qaer_delaa, qwtr3, qwtr4); + // map gas/aer/num arrays (mix-ratio and del=change) back to sub-area + // arrays + + if (do_map_gas_sub) { + for (int igas = 0; igas < 2; ++igas) { + const int l = lmap_gas(igas); + qsub4[l][jsub] = qgas4[igas] / fcvt_gas(igas); + for (int i = 0; i < nqtendaa(); ++i) + qsub_tendaa[l][i][jsub] = + qgas_delaa[igas][i] / (fcvt_gas(igas) * deltat); + } + } + for (int n = 0; n < num_modes; ++n) { + qsub4[n][jsub] = qnum4[n] / fcvt_num(); + for (int i = 0; i < nqtendaa(); ++i) + qsub_tendaa[n][i][jsub] = qnum_delaa[n][i] / (fcvt_num() * deltat); + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qsub4[iaer][jsub] = qaer4[iaer][n] / fcvt_aer(iaer); + for (int i = 0; i < nqtendaa(); ++i) + qsub_tendaa[iaer][i][jsub] = + qaer_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); + } + qaerwatsub4[n][jsub] = qwtr4[n] / fcvt_wtr(); + + if (iscldy_subarea[jsub]) { + qqcwsub4[n][jsub] = qnumcw4[n] / fcvt_num(); + for (int i = 0; i < nqqcwtendaa(); ++i) + qqcwsub_tendaa[n][i][jsub] = + qnumcw_delaa[n][i] / (fcvt_num() * deltat); + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qqcwsub4[iaer][jsub] = qaercw4[iaer][n] / fcvt_aer(iaer); + for (int i = 0; i < nqqcwtendaa(); ++i) + qqcwsub_tendaa[iaer][i][jsub] = + qaercw_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); + } + } + } + } + } +} + +} // anonymous namespace + +KOKKOS_INLINE_FUNCTION +void modal_aero_amicphys_intr( + const AmicPhysConfig& config, const int nstep, const Real deltat, const Real t, const Real pmid, const Real pdel, + const Real zm, const Real pblh, const Real qv, const Real cld, + Real q[gas_pcnst()], Real qqcw[gas_pcnst()], const Real q_pregaschem[gas_pcnst()], + const Real q_precldchem[gas_pcnst()], const Real qqcw_precldchem[gas_pcnst()], + Real q_tendbb[gas_pcnst()][nqtendbb()], Real qqcw_tendbb[gas_pcnst()][nqtendbb()], + Real dgncur_a[AeroConfig::num_modes()], + Real dgncur_awet[AeroConfig::num_modes()], + Real wetdens_host[AeroConfig::num_modes()], + Real qaerwat[AeroConfig::num_modes()]) { + + /* + nstep ! model time-step number + nqtendbb ! dimension for q_tendbb + nqqcwtendbb ! dimension f + deltat ! + q(ncol,pver,pcnstxx) ! current tracer mixing ratios (TMRs) + these values are updated (so out /= in) + *** MUST BE #/kmol-air for number + *** MUST BE mol/mol-air for mass + *** NOTE ncol dimension + qqcw(ncol,pver,pcnstxx) + like q but for cloud-borner tracers + these values are updated + q_pregaschem(ncol,pver,pcnstxx) ! q TMRs before gas-phase + chemistry q_precldchem(ncol,pver,pcnstxx) ! q TMRs before cloud + chemistry qqcw_precldchem(ncol,pver,pcnstxx) ! qqcw TMRs before cloud + chemistry q_tendbb(ncol,pver,pcnstxx,nqtendbb()) ! TMR tendencies for + box-model diagnostic output qqcw_tendbb(ncol,pver,pcnstx t(pcols,pver) ! + temperature at model levels (K) pmid(pcols,pver) ! pressure at model + level centers (Pa) pdel(pcols,pver) ! pressure thickness of levels + (Pa) zm(pcols,pver) ! altitude (above ground) at level centers (m) + pblh(pcols) ! planetary boundary layer depth (m) + qv(pcols,pver) ! specific humidity (kg/kg) + cld(ncol,pver) ! cloud fraction (-) *** NOTE ncol dimension + dgncur_a(pcols,pver,ntot_amode) + dgncur_awet(pcols,pver,ntot_amode) + ! dry & wet geo. mean dia. (m) of + number distrib. wetdens_host(pcols,pver,ntot_amode) ! interstitial + aerosol wet density (kg/m3) + + qaerwat(pcols,pver,ntot_amode aerosol water mixing ratio (kg/kg, + NOT mol/mol) + + */ + + // !DESCRIPTION: + // calculates changes to gas and aerosol TMRs (tracer mixing ratios) from + // gas-aerosol exchange (condensation/evaporation) + // growth from smaller to larger modes (renaming) due to both + // condensation and cloud chemistry + // new particle nucleation + // coagulation + // transfer of particles from hydrophobic modes to hydrophilic modes + // (aging) + // due to condensation and coagulation + // + // the incoming mixing ratios (q and qqcw) are updated before output + // + // !REVISION HISTORY: + // RCE 07.04.13: Adapted from earlier version of CAM5 modal aerosol + // routines + // for these processes + // + + static constexpr int num_modes = AeroConfig::num_modes(); + + // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios + // (TMRs, mol/mol or #/kmol) + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) + // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) + + // qsubN and qqcwsubN (N=1:4) are TMRs in sub-areas + // currently there are just clear and cloudy sub-areas + // the N=1:4 have same meanings as for qgcmN + + // q_coltendaa and qqcw_coltendaa are column-integrated tendencies + // for different processes, which are output to history + // the processes are condensation/evaporation (and associated aging), + // renaming, coagulation, and nucleation + + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendbb(); ++j) + q_tendbb[i][j] = 0.0, qqcw_tendbb[i][j] = 0.0; + + // get saturation mixing ratio + // call qsat( t(1:ncol,1:pver), pmid(1:ncol,1:pvnner), & + // ev_sat(1:ncol,1:pver), qv_sat(1:ncol,1:pver) ) + const Real epsqs = haero::Constants::weight_ratio_h2o_air; + // Saturation vapor pressure + const Real ev_sat = conversions::vapor_saturation_pressure_magnus(t, pmid); + // Saturation specific humidity + const Real qv_sat = epsqs * ev_sat / (pmid - (1 - epsqs) * ev_sat); + + const Real relhumgcm = haero::max(0.0, haero::min(1.0, qv / qv_sat)); + + // Set up cloudy/clear subareas inside a grid cell + int nsubarea, ncldy_subarea, jclea, jcldy; + bool iscldy_subarea[maxsubarea()]; + Real afracsub[maxsubarea()]; + Real relhumsub[maxsubarea()]; + Real qsub1[gas_pcnst()][maxsubarea()]; + Real qsub2[gas_pcnst()][maxsubarea()]; + Real qsub3[gas_pcnst()][maxsubarea()]; + Real qqcwsub1[gas_pcnst()][maxsubarea()]; + Real qqcwsub2[gas_pcnst()][maxsubarea()]; + Real qqcwsub3[gas_pcnst()][maxsubarea()]; + // aerosol water mixing ratios (mol/mol) + Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()]; + construct_subareas_1gridcell(cld, relhumgcm, // in + q_pregaschem, q_precldchem, // in + qqcw_precldchem, // in + q, qqcw, // in + nsubarea, ncldy_subarea, jclea, jcldy, // out + iscldy_subarea, afracsub, relhumsub, // out + qsub1, qsub2, qsub3, // out + qqcwsub1, qqcwsub2, qqcwsub3, qaerwatsub3, // out + qaerwat // in + ); + + // Initialize the "after-amicphys" values + Real qsub4[gas_pcnst()][maxsubarea()] = {}; + Real qqcwsub4[gas_pcnst()][maxsubarea()] = {}; + Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()] = {}; + + // + // start integration + // + Real dgn_a[num_modes], dgn_awet[num_modes], wetdens[num_modes]; + for (int n = 0; n < num_modes; ++n) { + dgn_a[n] = dgncur_a[n]; + dgn_awet[n] = dgncur_awet[n]; + wetdens[n] = haero::max(1000.0, wetdens_host[n]); + } + Real qsub_tendaa[gas_pcnst()][nqtendaa()][maxsubarea()] = {}; + Real qqcwsub_tendaa[gas_pcnst()][nqqcwtendaa()][maxsubarea()] = {}; + mam_amicphys_1gridcell(config, nstep, deltat, + nsubarea, ncldy_subarea, iscldy_subarea, afracsub, t, + pmid, pdel, zm, pblh, relhumsub, dgn_a, dgn_awet, + wetdens, qsub1, qsub2, qqcwsub2, qsub3, qqcwsub3, + qaerwatsub3, qsub4, qqcwsub4, qaerwatsub4, qsub_tendaa, + qqcwsub_tendaa); + + // + // form new grid-mean mix-ratios + Real qgcm4[gas_pcnst()]; + Real qgcm_tendaa[gas_pcnst()][nqtendaa()]; + Real qaerwatgcm4[num_modes]; + if (nsubarea == 1) { + for (int i = 0; i < gas_pcnst(); ++i) + qgcm4[i] = qsub4[i][0]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendaa(); ++j) + qgcm_tendaa[i][j] = qsub_tendaa[i][j][0]; + for (int i = 0; i < num_modes; ++i) + qaerwatgcm4[i] = qaerwatsub4[i][0]; + } else { + for (int i = 0; i < gas_pcnst(); ++i) + qgcm4[i] = 0.0; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendaa(); ++j) + qgcm_tendaa[i][j] = 0.0; + for (int n = 0; n < nsubarea; ++n) { + for (int i = 0; i < gas_pcnst(); ++i) + qgcm4[i] += qsub4[i][n] * afracsub[n]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendaa(); ++j) + qgcm_tendaa[i][j] = + qgcm_tendaa[i][j] + qsub_tendaa[i][j][n] * afracsub[n]; + } + for (int i = 0; i < num_modes; ++i) + // for aerosol water use the clear sub-area value + qaerwatgcm4[i] = qaerwatsub4[i][jclea - 1]; + } + Real qqcwgcm4[gas_pcnst()]; + Real qqcwgcm_tendaa[gas_pcnst()][nqqcwtendaa()]; + if (ncldy_subarea <= 0) { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] = haero::max(0.0, qqcw[i]); + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] = 0.0; + } else if (nsubarea == 1) { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] = qqcwsub4[i][0]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] = qqcwsub_tendaa[i][j][0]; + } else { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] = 0.0; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] = 0.0; + for (int n = 0; n < nsubarea; ++n) { + if (iscldy_subarea[n]) { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] += qqcwsub4[i][n] * afracsub[n]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] += qqcwsub_tendaa[i][j][n] * afracsub[n]; + } + } + } + + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (lmapcc_all(lmz) > 0) { + // HW, to ensure non-negative + q[lmz] = haero::max(qgcm4[lmz], 0.0); + if (lmapcc_all(lmz) >= lmapcc_val_aer()) { + // HW, to ensure non-negative + qqcw[lmz] = haero::max(qqcwgcm4[lmz], 0.0); + } + } + } + for (int i = 0; i < gas_pcnst(); ++i) { + if (iqtend_cond() < nqtendbb()) + q_tendbb[i][iqtend_cond()] = qgcm_tendaa[i][iqtend_cond()]; + if (iqtend_rnam() < nqtendbb()) + q_tendbb[i][iqtend_rnam()] = qgcm_tendaa[i][iqtend_rnam()]; + if (iqtend_nnuc() < nqtendbb()) + q_tendbb[i][iqtend_nnuc()] = qgcm_tendaa[i][iqtend_nnuc()]; + if (iqtend_coag() < nqtendbb()) + q_tendbb[i][iqtend_coag()] = qgcm_tendaa[i][iqtend_coag()]; + if (iqqcwtend_rnam() < nqqcwtendbb()) + qqcw_tendbb[i][iqqcwtend_rnam()] = qqcwgcm_tendaa[i][iqqcwtend_rnam()]; + } + for (int i = 0; i < num_modes; ++i) + qaerwat[i] = qaerwatgcm4[i]; +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/online_emission.hpp b/components/eamxx/src/physics/mam/online_emission.hpp index b844f5b18e6..c8318dcc05c 100644 --- a/components/eamxx/src/physics/mam/online_emission.hpp +++ b/components/eamxx/src/physics/mam/online_emission.hpp @@ -63,12 +63,26 @@ template struct onlineEmissions { // --------------------------------------------------------------------------- // Online emissions routines // --------------------------------------------------------------------------- - void init_from_input_file(const ekat::ParameterList &m_params); - void transfer_to_cflux(const onlineEmissData &data, - const std::map idx_map, - view_2d &fluxes); + void init_from_input_file(const ekat::ParameterList ¶ms) { + const int nspec = online_emis_data.nspec; + const int ncols = online_emis_data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nspec); + // Read from input file + // FIXME: currently reading a single placeholder scalar--should be + // ncols-sized array when we know what the input data looks like + for (int ispec = 0; ispec < nspec; ++ispec) { + Real init_cond_val = + params.get(online_emis_data.root_IC_str + online_emis_data.spec_names[ispec]); + // TODO: is this overkill?--i.e., would a mirror/deep_copy make more sense? + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int jcol = team.league_rank(); // column index + online_emis_data.flux_data(ispec, jcol) = init_cond_val; + }); + } + } // end init_from_input_file() }; // struct onlineEmissions } // namespace scream::mam_coupling #endif // ONLINE_EMISSION_HPP - -#include "online_emission_impl.hpp" diff --git a/components/eamxx/src/physics/mam/online_emission_impl.hpp b/components/eamxx/src/physics/mam/online_emission_impl.hpp deleted file mode 100644 index 1f0ac91a854..00000000000 --- a/components/eamxx/src/physics/mam/online_emission_impl.hpp +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef ONLINE_EMISSION_IMPL_HPP -#define ONLINE_EMISSION_IMPL_HPP - -namespace scream::mam_coupling { -template -void onlineEmissions::init_from_input_file(const ekat::ParameterList ¶ms) { - // FIXME: move this out to the onlineEmissions struct to avoid extra work - // by doing it again below? - const int nspec = online_emis_data.nspec; - const int ncols = online_emis_data.ncols; - using ExeSpace = typename KT::ExeSpace; - using ESU = ekat::ExeSpaceUtils; - const auto policy = ESU::get_default_team_policy(ncols, nspec); - // Read from input file - // FIXME: currently reading a single placeholder scalar--should be - // ncols-sized array when we know what the input data looks like - for (int ispec = 0; ispec < nspec; ++ispec) { - Real init_cond_val = - params.get(online_emis_data.root_IC_str + online_emis_data.spec_names[ispec]); - // TODO: is this overkill?--i.e., would a mirror/deep_copy make more sense? - Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const MemberType &team) { - const int jcol = team.league_rank(); // column index - online_emis_data.flux_data(ispec, jcol) = init_cond_val; - }); - } -} // end init_from_input_file() - -template -void onlineEmissions::transfer_to_cflux( - const onlineEmissData &data, const std::map idx_map, - view_2d &fluxes) { - const int nspec = data.nspec; - const int ncols = data.ncols; - using ExeSpace = typename KT::ExeSpace; - using ESU = ekat::ExeSpaceUtils; - const auto policy = ESU::get_default_team_policy(ncols, nspec); - - Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const MemberType &team) { - const int jcol = team.league_rank(); // column index - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nspec), [&](const int ispec) { - auto s_idx = idx_map.at(data.spec_names[ispec]); - data.cfluxes(jcol, s_idx) = data.flux_data(ispec, jcol); - }); - }); - Kokkos::deep_copy(fluxes, data.cfluxes); -} -} // namespace scream::mam_coupling - -#endif // ONLINE_EMISSION_IMPL_HPP From 405ce81346acd04960adbcb5acc8a6863045e77e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 26 Oct 2024 12:38:49 -0700 Subject: [PATCH 326/366] Adds sst variable from the ocean model in EAMxx via coupler --- .../src/control/atmosphere_surface_coupling_importer.cpp | 3 +++ components/eamxx/src/mct_coupling/scream_cpl_indices.F90 | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index ee3e21e7461..f0eca885ddc 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -57,6 +57,9 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr("fv", scalar2d_layout, m/s, grid_name); // Aerodynamical resistance add_field("ram1", scalar2d_layout, s/m, grid_name); + // Sea surface temperature [K] + //FIXME: Verify the units + add_field("sst", scalar2d_layout, K, grid_name); } // ========================================================================================= void SurfaceCouplingImporter::setup_surface_coupling_data(const SCDataManager &sc_data_manager) diff --git a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 index dc5be4de373..2627ac722fd 100644 --- a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 +++ b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 @@ -6,7 +6,7 @@ module scream_cpl_indices private ! Focus only on the ones that scream imports/exports (subsets of x2a and a2x) - integer, parameter, public :: num_scream_imports = 19 + integer, parameter, public :: num_scream_imports = 20 integer, parameter, public :: num_scream_exports = 17 integer, public :: num_cpl_imports, num_cpl_exports, import_field_size, export_field_size @@ -92,6 +92,7 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_field_names(17) = 'icefrac' import_field_names(18) = 'fv' import_field_names(19) = 'ram1' + import_field_names(20) = 'sst' ! CPL indices import_cpl_indices(1) = mct_avect_indexra(x2a,'Sx_avsdr') @@ -113,6 +114,7 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_cpl_indices(17) = mct_avect_indexra(x2a,'Sf_ifrac') import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_fv') import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_ram1') + import_cpl_indices(20) = mct_avect_indexra(x2a,'So_t') ! Vector components import_vector_components(11) = 0 From e38b45bd7caf336280e36a61cfa5298d3960be61 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 26 Oct 2024 13:22:22 -0700 Subject: [PATCH 327/366] Adds dstfluxes (4 values per col) from coupler to eamxx --- .../atmosphere_surface_coupling_importer.cpp | 52 ++++++++++--------- .../src/mct_coupling/scream_cpl_indices.F90 | 14 ++++- ...and_online_emissions_process_interface.cpp | 2 +- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index f0eca885ddc..e87bebd34e1 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -28,38 +28,42 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr("sfc_alb_dir_vis", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dir_nir", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dif_vis", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dif_nir", scalar2d_layout, nondim, grid_name); - add_field("surf_lw_flux_up", scalar2d_layout, W/m2, grid_name); - add_field("surf_sens_flux", scalar2d_layout, W/m2, grid_name); - add_field("surf_evap", scalar2d_layout, kg/m2/s, grid_name); - add_field("surf_mom_flux", vector2d_layout, N/m2, grid_name); - add_field("surf_radiative_T", scalar2d_layout, K, grid_name); - add_field("T_2m", scalar2d_layout, K, grid_name); - add_field("qv_2m", scalar2d_layout, kg/kg, grid_name); - add_field("wind_speed_10m", scalar2d_layout, m/s, grid_name); - add_field("snow_depth_land", scalar2d_layout, m, grid_name); - add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - add_field("landfrac", scalar2d_layout, nondim, grid_name); - add_field("icefrac", scalar2d_layout, nondim, grid_name); + const FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + const FieldLayout vector2d = m_grid->get_2d_vector_layout(2); + const FieldLayout vector4d = m_grid->get_2d_vector_layout(4); + + add_field("sfc_alb_dir_vis", scalar2d, nondim, grid_name); + add_field("sfc_alb_dir_nir", scalar2d, nondim, grid_name); + add_field("sfc_alb_dif_vis", scalar2d, nondim, grid_name); + add_field("sfc_alb_dif_nir", scalar2d, nondim, grid_name); + add_field("surf_lw_flux_up", scalar2d, W/m2, grid_name); + add_field("surf_sens_flux", scalar2d, W/m2, grid_name); + add_field("surf_evap", scalar2d, kg/m2/s, grid_name); + add_field("surf_mom_flux", vector2d, N/m2, grid_name); + add_field("surf_radiative_T", scalar2d, K, grid_name); + add_field("T_2m", scalar2d, K, grid_name); + add_field("qv_2m", scalar2d, kg/kg, grid_name); + add_field("wind_speed_10m", scalar2d, m/s, grid_name); + add_field("snow_depth_land", scalar2d, m, grid_name); + add_field("ocnfrac", scalar2d, nondim, grid_name); + add_field("landfrac", scalar2d, nondim, grid_name); + add_field("icefrac", scalar2d, nondim, grid_name); // Friction velocity [m/s] - add_field("fv", scalar2d_layout, m/s, grid_name); + add_field("fv", scalar2d, m/s, grid_name); // Aerodynamical resistance - add_field("ram1", scalar2d_layout, s/m, grid_name); + add_field("ram1", scalar2d, s/m, grid_name); // Sea surface temperature [K] //FIXME: Verify the units - add_field("sst", scalar2d_layout, K, grid_name); + add_field("sst", scalar2d, K, grid_name); + //dust fluxes [kg/m^2/s]: Four flux values for eacch column + add_field("dstflx", vector4d, kg/m2/s, grid_name); + } // ========================================================================================= void SurfaceCouplingImporter::setup_surface_coupling_data(const SCDataManager &sc_data_manager) diff --git a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 index 2627ac722fd..15df178d29e 100644 --- a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 +++ b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 @@ -6,7 +6,7 @@ module scream_cpl_indices private ! Focus only on the ones that scream imports/exports (subsets of x2a and a2x) - integer, parameter, public :: num_scream_imports = 20 + integer, parameter, public :: num_scream_imports = 24 integer, parameter, public :: num_scream_exports = 17 integer, public :: num_cpl_imports, num_cpl_exports, import_field_size, export_field_size @@ -93,6 +93,10 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_field_names(18) = 'fv' import_field_names(19) = 'ram1' import_field_names(20) = 'sst' + import_field_names(21) = 'dstflx' + import_field_names(22) = 'dstflx' + import_field_names(23) = 'dstflx' + import_field_names(24) = 'dstflx' ! CPL indices import_cpl_indices(1) = mct_avect_indexra(x2a,'Sx_avsdr') @@ -114,7 +118,15 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_cpl_indices(17) = mct_avect_indexra(x2a,'Sf_ifrac') import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_fv') import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_ram1') + !sst import_cpl_indices(20) = mct_avect_indexra(x2a,'So_t') + !dust fluxes + import_cpl_indices(21) = mct_avect_indexra(x2a,'Fall_flxdst1') + import_cpl_indices(22) = mct_avect_indexra(x2a,'Fall_flxdst2') + import_cpl_indices(23) = mct_avect_indexra(x2a,'Fall_flxdst3') + import_cpl_indices(24) = mct_avect_indexra(x2a,'Fall_flxdst4') + + ! Vector components import_vector_components(11) = 0 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index c84bb803440..5e44fd1e35e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -51,7 +51,7 @@ void MAMSrfOnlineEmiss::set_grids( // and then bundle the online emissions computations as tendencies into the // Computed constituent_fluxes field // ------------------------------------------------------------- - static constexpr Units m2(m * m, "m2"); + constexpr auto m2 = pow(m, 2); // Constituent fluxes of species in [kg/m2/s] add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); From 13f6cfcfb939b7f90a42ce16b553d9fb41520bb2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 27 Oct 2024 22:48:21 -0700 Subject: [PATCH 328/366] Adds all input variables for online emissions into the interface cpp --- .../atmosphere_surface_coupling_importer.cpp | 1 - ...and_online_emissions_process_interface.cpp | 223 +++++++++++++----- ...and_online_emissions_process_interface.hpp | 68 ++++-- .../single-process/mam/emissions/input.yaml | 4 + 4 files changed, 217 insertions(+), 79 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index e87bebd34e1..2c3360a3b4f 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -59,7 +59,6 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr("ram1", scalar2d, s/m, grid_name); // Sea surface temperature [K] - //FIXME: Verify the units add_field("sst", scalar2d, K, grid_name); //dust fluxes [kg/m^2/s]: Four flux values for eacch column add_field("dstflx", vector4d, kg/m2/s, grid_name); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 5e44fd1e35e..b495c6614ef 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -19,41 +19,101 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, // ================================================================ void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { - grid_ = grids_manager->get_grid("Physics"); + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; + constexpr auto m2 = pow(m, 2); + constexpr auto s2 = pow(s, 2); + constexpr auto q_unit = kg / kg; // units of mass mixing ratios of tracers + constexpr auto n_unit = 1 / kg; // units of number mixing ratios of tracers + constexpr auto nondim = ekat::units::Units::nondimensional(); - static constexpr int pcnst = mam4::aero_model::pcnst; - const FieldLayout scalar2d_pcnct = - grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); - // auto vector3d_mid = grid_->get_3d_vector_layout(true, 2); + const FieldLayout scalar2d = grid_->get_2d_scalar_layout(); + const FieldLayout scalar3d_m = grid_->get_3d_scalar_layout(true); // mid + const FieldLayout scalar3d_i = grid_->get_3d_scalar_layout(false); // int + + // For U and V components of wind + const FieldLayout vector3d = grid_->get_3d_vector_layout(true, 2); + + // For components of dust flux + const FieldLayout vector4d = grid_->get_2d_vector_layout(4); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- + + // ----------- Atmospheric quantities ------------- + + // -- Variables required for building DS to compute z_mid -- + // Specific humidity [kg/kg] + // FIXME: Comply with add_tracer calls + add_field("qv", scalar3d_m, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_m, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_m, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_m, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_m, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_m, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints (Require only for building + // DS) + add_field("omega", scalar3d_m, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_m, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_i, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_m, Pa, grid_name); - // FIXME: online emissions requires the following quantities for the - // OnlineEmissionsData struct: {surface_temp, u_bottom, v_bottom, z_bottom, - // ocean_frac} - // // Temperature - // add_field("T_mid", scalar3d_layout_mid, K, grid_name); - // add_field("horiz_winds", vector3d_mid, m/s, grid_name); - // vertical wind? - // ocean_frac? + // Planetary boundary layer height [m] + add_field("pbl_height", scalar2d, m, grid_name); + + // Surface geopotential [m2/s2] + add_field("phis", scalar2d, m2 / s2, grid_name); + + //----------- Variables from microphysics scheme ------------- + + // Total cloud fraction [fraction] (Require only for building DS) + add_field("cldfrac_tot", scalar3d_m, nondim, grid_name); + + // -- Variables required for online dust and sea salt emissions -- + + // U and V components of the wind[m/s] + add_field("horiz_winds", vector3d, m / s, grid_name); + + // Sea surface temperature [K] + add_field("sst", scalar2d, K, grid_name); + + // dust fluxes [kg/m^2/s]: Four flux values for eacch column + add_field("dstflx", vector4d, kg / m2 / s, grid_name); // ------------------------------------------------------------- - // These variables are "Computed" or outputs for the process - // FIXME: this should likely be an updated field, since online emissisons - // expects input values for constituent_fluxes - // NOTE: the other option is that we do something like: - // add_field("constituent_fluxes_input", scalar2d_pcnct, - // kg / m2 / s, grid_name); - // and then bundle the online emissions computations as tendencies into the - // Computed constituent_fluxes field + // These variables are "Updated" or input-outputs for the process // ------------------------------------------------------------- - constexpr auto m2 = pow(m, 2); + + constexpr int pcnst = mam4::aero_model::pcnst; + const FieldLayout vector2d_pcnst = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + // Constituent fluxes of species in [kg/m2/s] - add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, + // FIXME: confirm if it is Updated or Computed + add_field("constituent_fluxes", vector2d_pcnst, kg / m2 / s, grid_name); // Surface emissions remapping file @@ -66,20 +126,20 @@ void MAMSrfOnlineEmiss::set_grids( //-------------------------------------------------------------------- srf_emiss_ dms; // File name, name and sectors - dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); dms.species_name = "dms"; - dms.sectors = {"DMS"}; - srf_emiss_species_.push_back(dms); // add to the vector + dms.sectors = {"DMS"}; + srf_emiss_species_.push_back(dms); // add to the vector //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- srf_emiss_ so2; // File name, name and sectors - so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); + so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); so2.species_name = "so2"; - so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; - srf_emiss_species_.push_back(so2); // add to the vector + so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(so2); // add to the vector //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures @@ -88,8 +148,8 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors bc_a4.data_file = m_params.get("srf_emis_specifier_for_bc_a4"); bc_a4.species_name = "bc_a4"; - bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - srf_emiss_species_.push_back(bc_a4); // add to the vector + bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(bc_a4); // add to the vector //-------------------------------------------------------------------- // Init num_a1 srf emiss data structures @@ -98,9 +158,9 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors num_a1.data_file = m_params.get("srf_emis_specifier_for_num_a1"); num_a1.species_name = "num_a1"; - num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", - "num_a1_SO4_WST"}; - srf_emiss_species_.push_back(num_a1); // add to the vector + num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", + "num_a1_SO4_WST"}; + srf_emiss_species_.push_back(num_a1); // add to the vector //-------------------------------------------------------------------- // Init num_a2 srf emiss data structures @@ -109,8 +169,8 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors num_a2.data_file = m_params.get("srf_emis_specifier_for_num_a2"); num_a2.species_name = "num_a2"; - num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; - srf_emiss_species_.push_back(num_a2); // add to the vector + num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + srf_emiss_species_.push_back(num_a2); // add to the vector //-------------------------------------------------------------------- // Init num_a4 srf emiss data structures @@ -119,12 +179,12 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors num_a4.data_file = m_params.get("srf_emis_specifier_for_num_a4"); num_a4.species_name = "num_a4"; - num_a4.sectors = { - "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", - "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", - "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", - "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; - srf_emiss_species_.push_back(num_a4); // add to the vector + num_a4.sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + srf_emiss_species_.push_back(num_a4); // add to the vector //-------------------------------------------------------------------- // Init pom_a4 srf emiss data structures @@ -134,7 +194,7 @@ void MAMSrfOnlineEmiss::set_grids( pom_a4.data_file = m_params.get("srf_emis_specifier_for_pom_a4"); pom_a4.species_name = "pom_a4"; pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - srf_emiss_species_.push_back(pom_a4); // add to the vector + srf_emiss_species_.push_back(pom_a4); // add to the vector //-------------------------------------------------------------------- // Init so4_a1 srf emiss data structures @@ -143,7 +203,7 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors so4_a1.data_file = m_params.get("srf_emis_specifier_for_so4_a1"); so4_a1.species_name = "so4_a1"; - so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; + so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; srf_emiss_species_.push_back(so4_a1); //-------------------------------------------------------------------- @@ -153,20 +213,20 @@ void MAMSrfOnlineEmiss::set_grids( // File name, name and sectors so4_a2.data_file = m_params.get("srf_emis_specifier_for_so4_a2"); so4_a2.species_name = "so4_a2"; - so4_a2.sectors = {"RCO", "TRA"}; + so4_a2.sectors = {"RCO", "TRA"}; srf_emiss_species_.push_back(so4_a2); //-------------------------------------------------------------------- // Init data structures to read and interpolate //-------------------------------------------------------------------- - for (srf_emiss_ &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::init_srf_emiss_objects( ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, // output ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, ispec_srf.data_out_, ispec_srf.dataReader_); } -} // set_grid ends +} // set_grid ends // ================================================================ // REQUEST_BUFFER_SIZE_IN_BYTES @@ -184,9 +244,9 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= - requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); @@ -199,6 +259,42 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { // INITIALIZE_IMPL // ================================================================ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- + + // Populate the wet atmosphere state with views from fields + wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + // --------------------------------------------------------------- // Output fields // --------------------------------------------------------------- @@ -223,10 +319,10 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { //-------------------------------------------------------------------- // Update surface emissions from file //-------------------------------------------------------------------- - for (srf_emiss_ &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::update_srfEmiss_data_from_file( ispec_srf.dataReader_, timestamp(), curr_month, *ispec_srf.horizInterp_, - ispec_srf.data_end_); // output + ispec_srf.data_end_); // output } //-------------------------------------------------------------------- @@ -238,15 +334,17 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- - preprocess_.initialize(constituent_fluxes_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_, constituent_fluxes_); -} // end initialize_impl() +} // end initialize_impl() // ================================================================ // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Zero-out output + // FIXME: Find out if we do it in the fortran code + // if we do, this should be a "Computed" field Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation @@ -256,7 +354,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Interpolate srf emiss data //-------------------------------------------------------------------- - for (srf_emiss_ &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { // Update TimeState, note the addition of dt ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); @@ -279,7 +377,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // modify units from molecules/cm2/s to kg/m2/s auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - auto constituent_fluxes = this->constituent_fluxes_; + auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units @@ -289,7 +387,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); }); - } // for loop for species + } // for loop for species auto &online_data = online_emissions.online_emis_data; // TODO: check that units are consistent with srf emissions! @@ -310,13 +408,14 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // which is a chunk-wise variable at this point // (see: physpkg.F90:{1605 & 1327}) otherwise grabs the end/bottom // col-value once inside aero_model_emissions() - view_1d fluxes_col = Kokkos::subview(online_data.cfluxes, icol, Kokkos::ALL()); + view_1d fluxes_col = + Kokkos::subview(online_data.cfluxes, icol, Kokkos::ALL()); mam4::aero_model_emissions::aero_model_emissions(fluxes_col); }); // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes // in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert Kokkos::deep_copy(constituent_fluxes_, online_data.cfluxes); Kokkos::fence(); -} // run_impl ends +} // run_impl ends // ============================================================================= -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 80c2efcccd7..97fbbda5aca 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -12,7 +12,6 @@ // For declaring surface and online emission class derived from atm process // class #include - #include namespace scream { @@ -20,19 +19,31 @@ namespace scream { // The process responsible for handling MAM4 surface and online emissions. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { - using KT = ekat::KokkosTypes; - using view_1d = typename KT::template view_1d; - using view_2d = typename KT::template view_2d; + using KT = ekat::KokkosTypes; + using view_1d = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; + using const_view_1d = typename KT::template view_1d; + using const_view_2d = typename KT::template view_2d; // number of horizontal columns and vertical levels int ncol_, nlev_; + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + // buffer for sotring temporary variables mam_coupling::Buffer buffer_; // physics grid for column information std::shared_ptr grid_; + // Sea surface temoerature [K] + const_view_1d sst_; + + // Dust fluxes (four values for each col) [kg/m2/s] + const_view_2d dust_fluxes_; + // Constituent fluxes of species in [kg/m2/s] view_2d constituent_fluxes_; @@ -40,11 +51,11 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { view_1d fluxes_in_mks_units_; // Unified atomic mass unit used for unit conversion (BAD constant) - static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu + static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu -public: + public: using srfEmissFunc = mam_coupling::srfEmissFunctions; - using onlineEmiss = mam_coupling::onlineEmissions; + using onlineEmiss = mam_coupling::onlineEmissions; // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -60,8 +71,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::string name() const { return "mam_srf_online_emissions"; } // grid - void - set_grids(const std::shared_ptr grids_manager) override; + void set_grids( + const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -81,14 +92,39 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct Preprocess { Preprocess() = default; // on host: initializes preprocess functor with necessary state data - void initialize(const view_2d &constituent_fluxes) { + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::DryAtmosphere &dry_atm, + const view_2d &constituent_fluxes) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + dry_atm_pre_ = dry_atm; constituent_fluxes_pre_ = constituent_fluxes; } + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, i); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + } // Preprocess operator() + // local variables for preprocess struct - view_2d constituent_fluxes_pre_; - }; // MAMSrfOnlineEmiss::Preprocess + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; -private: + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + view_2d constituent_fluxes_pre_; + }; // MAMSrfOnlineEmiss::Preprocess + private: // preprocessing scratch pad Preprocess preprocess_; @@ -131,8 +167,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; -}; // MAMSrfOnlineEmiss +}; // MAMSrfOnlineEmiss -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 7a3dfb43f22..b337ce3a503 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -69,6 +69,10 @@ initial_conditions: #we should get the following variables from other processes pbl_height : 1.0 + dstflx : [ -0.99391637990504811E-011, -0.53350997938896468E-010, -0.12510222703847123E-009, -0.11784238261270793E-009] + ocnfrc: 0.99999999999999989E+000 + sst: 0.30196620710190155E+003 + cldfrac_tot: 0.138584624960092 # The parameters for I/O control Scorpio: From 52afd72edeea8bc59545330623a551cca2fd17fd Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 28 Oct 2024 11:17:35 -0700 Subject: [PATCH 329/366] Commits changes in submodule MAM4xx --- ...and_online_emissions_process_interface.cpp | 76 +++++++++++-------- externals/mam4xx | 2 +- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index b495c6614ef..d2221690c5d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -342,17 +342,43 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + // Zero-out output // FIXME: Find out if we do it in the fortran code - // if we do, this should be a "Computed" field + // if we do, this should be a "Computed" field, + // ALSO this code should in the preprocessor struct now Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); - // Gather time and state information for interpolation - auto ts = timestamp() + dt; + //-------------------------------------------------------------------- + // Online emissions from dust and sea salt + //-------------------------------------------------------------------- + + const const_view_2d dstflx = get_field_in("dstflx").get_view(); + auto constituent_fluxes = this->constituent_fluxes_; + // TODO: check that units are consistent with srf emissions! + // copy current values to online-emissions-local version + // TODO: potentially combine with below parfor(icol) loop? + Kokkos::parallel_for( + "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); + mam4::aero_model_emissions::aero_model_emissions(dstflx, fluxes_col); + }); + // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes + // in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert + //Kokkos::deep_copy(constituent_fluxes_, online_data.cfluxes); + //Kokkos::fence(); //-------------------------------------------------------------------- - // Interpolate srf emiss data + // Interpolate srf emiss data read in from emissions files //-------------------------------------------------------------------- + // Gather time and state information for interpolation + auto ts = timestamp() + dt; for(srf_emiss_ &ispec_srf : srf_emiss_species_) { // Update TimeState, note the addition of dt @@ -377,7 +403,6 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // modify units from molecules/cm2/s to kg/m2/s auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units @@ -388,34 +413,19 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); }); } // for loop for species - - auto &online_data = online_emissions.online_emis_data; - // TODO: check that units are consistent with srf emissions! - // copy current values to online-emissions-local version - Kokkos::deep_copy(online_data.cfluxes, constituent_fluxes_); - // TODO: potentially combine with above parfor(icol) loop? - Kokkos::parallel_for( - "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - // NOTE: calling aero_model_emissions() this way hides the fact that - // some hard-coded data is initialized within mam4::aero_model_emissions - // some of these values definitely need to be read from here column-wise - // the structs.{values} holding the above are: - // SeasaltEmissionsData.{mpoly, mprot, mlip} - // DustEmissionsData.dust_dmt_vwr - // OnlineEmissionsData.{dust_flux_in, surface_temp, u_bottom, - // v_bottom, z_bottom, ocean_frac} - // NOTE: fortran mam4 gets dust_flux_in and ocean_frac from cam_in, - // which is a chunk-wise variable at this point - // (see: physpkg.F90:{1605 & 1327}) otherwise grabs the end/bottom - // col-value once inside aero_model_emissions() - view_1d fluxes_col = - Kokkos::subview(online_data.cfluxes, icol, Kokkos::ALL()); - mam4::aero_model_emissions::aero_model_emissions(fluxes_col); - }); - // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes - // in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert - Kokkos::deep_copy(constituent_fluxes_, online_data.cfluxes); - Kokkos::fence(); } // run_impl ends // ============================================================================= } // namespace scream + +// NOTE: calling aero_model_emissions() this way hides the fact that +// some hard-coded data is initialized within mam4::aero_model_emissions +// some of these values definitely need to be read from here column-wise +// the structs.{values} holding the above are: +// SeasaltEmissionsData.{mpoly, mprot, mlip} +// DustEmissionsData.dust_dmt_vwr +// OnlineEmissionsData.{dust_flux_in, surface_temp, u_bottom, +// v_bottom, z_bottom, ocean_frac} +// NOTE: fortran mam4 gets dust_flux_in and ocean_frac from cam_in, +// which is a chunk-wise variable at this point +// (see: physpkg.F90:{1605 & 1327}) otherwise grabs the end/bottom +// col-value once inside aero_model_emissions() \ No newline at end of file diff --git a/externals/mam4xx b/externals/mam4xx index 6bc2697f84b..c837e32f0f3 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 6bc2697f84b38503cc8fe638a6281ea107180e40 +Subproject commit c837e32f0f3ee63d2a7b50bd5bcf9e9b29845265 From 16a12172565d78664dc9eef5d6b90151e9a4b0dc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 28 Oct 2024 18:07:49 -0700 Subject: [PATCH 330/366] EAMxx:Adds soil erodibility file read and sent to online emission read --- .../cime_config/namelist_defaults_scream.xml | 9 ++ ...and_online_emissions_process_interface.cpp | 48 +++++- ...and_online_emissions_process_interface.hpp | 5 + .../mam/readfiles/soil_erodibility.hpp | 57 +++++++ .../mam/readfiles/soil_erodibility_impl.hpp | 145 ++++++++++++++++++ .../single-process/mam/emissions/input.yaml | 2 + 6 files changed, 260 insertions(+), 6 deletions(-) create mode 100644 components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp create mode 100644 components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 195f46847ef..40211361376 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -371,7 +371,16 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc +<<<<<<< HEAD +======= + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/dst_ne30pg2_c20241028.nc + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/dst_ne4pg2_c20241028.nc +>>>>>>> EAMxx:Adds soil erodibility file read and sent to online emission read ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index d2221690c5d..f6ce5dc9f82 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,8 +1,13 @@ -//#include + #include +// For reading soil erodibility file +#include + namespace scream { +using soilErodibilityFunc = + soil_erodibility::soilErodibilityFunctions; // ================================================================ // Constructor // ================================================================ @@ -225,7 +230,27 @@ void MAMSrfOnlineEmiss::set_grids( // output ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, ispec_srf.data_out_, ispec_srf.dataReader_); - } + } // srf emissions file read init + + // ------------------------------------------------------------- + // setup to enable reading soil erodibility file + // ------------------------------------------------------------- + + const std::string soil_erodibility_data_file = + m_params.get("soil_erodibility_file"); + // /compyfs/inputdata/atm/cam/dst/dst_1.9x2.5_c090203.nc + + // Field to be read from file + const std::string field_name = "mbl_bsn_fct_geo"; + + // Dimensions of the filed + const std::string dim_name1 = "ncol"; + + // initialize the file read + soilErodibilityFunc::init_soil_erodibility_file_read( + ncol_, field_name, dim_name1, grid_, soil_erodibility_data_file, + srf_map_file, horizInterp_, dataReader_); // output + } // set_grid ends // ================================================================ @@ -325,6 +350,15 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { ispec_srf.data_end_); // output } + //----------------------------------------------------------------- + // Read Soil erodibility data + //----------------------------------------------------------------- + // This data is time-independent, we read all data here for the + // entire simulation + soilErodibilityFunc::update_soil_erodibility_data_from_file( + dataReader_, *horizInterp_, + soil_erodibility_); // output + //-------------------------------------------------------------------- // Initialize online emissions from file //-------------------------------------------------------------------- @@ -360,19 +394,21 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- const const_view_2d dstflx = get_field_in("dstflx").get_view(); - auto constituent_fluxes = this->constituent_fluxes_; + auto constituent_fluxes = this->constituent_fluxes_; + auto soil_erodibility = this->soil_erodibility_; // TODO: check that units are consistent with srf emissions! // copy current values to online-emissions-local version // TODO: potentially combine with below parfor(icol) loop? Kokkos::parallel_for( "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); - mam4::aero_model_emissions::aero_model_emissions(dstflx, fluxes_col); + mam4::aero_model_emissions::aero_model_emissions( + dstflx, soil_erodibility, fluxes_col); }); // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes // in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert - //Kokkos::deep_copy(constituent_fluxes_, online_data.cfluxes); - //Kokkos::fence(); + // Kokkos::deep_copy(constituent_fluxes_, online_data.cfluxes); + // Kokkos::fence(); //-------------------------------------------------------------------- // Interpolate srf emiss data read in from emissions files diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 97fbbda5aca..c1915cd81ba 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -53,6 +53,11 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // Unified atomic mass unit used for unit conversion (BAD constant) static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu + // For reading soil erodibility file + std::shared_ptr horizInterp_; + std::shared_ptr dataReader_; + const_view_1d soil_erodibility_; + public: using srfEmissFunc = mam_coupling::srfEmissFunctions; using onlineEmiss = mam_coupling::onlineEmissions; diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp new file mode 100644 index 00000000000..9be7c0f817b --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp @@ -0,0 +1,57 @@ +#ifndef SOIL_ERODIBILITY_HPP +#define SOIL_ERODIBILITY_HPP + +// For AtmosphereInput +#include "share/io/scorpio_input.hpp" + +namespace scream { +namespace soil_erodibility { + +template +struct soilErodibilityFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using const_view_1d = typename KT::template view_1d; + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + // Soil erodibility routines + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &soilErodibility_data_file, const std::string &map_file, + const std::string &field_name, const std::string &dim_name1); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + static std::shared_ptr create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + static void update_soil_erodibility_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_1d &input); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + static void init_soil_erodibility_file_read( + const int ncol, const std::string field_name, const std::string dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &SoilErodibilityHorizInterp, + std::shared_ptr &SoilErodibilityDataReader); + +}; // struct soilErodilityFunctions + +} // namespace soil_erodibilty +} // namespace scream +#endif // SOIL_ERODIBILITY_HPP + +#include "soil_erodibility_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp new file mode 100644 index 00000000000..2f6a5da6888 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp @@ -0,0 +1,145 @@ +#ifndef SOIL_ERODIBILITY_IMPL_HPP +#define SOIL_ERODIBILITY_IMPL_HPP + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" + +namespace scream { +namespace soil_erodibility { + +template +std::shared_ptr +soilErodibilityFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const std::string &field_name, const std::string &dim_name1) { + + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, dim_name1); + + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("soil_erodibility_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen soil erodibility " + "data to fit the model. We only allow\n" + " soil erodibility data to be at the same or " + "coarser resolution as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG(map_file != "", + "ERROR: soil erodibility data is on a different grid " + "than the model one,\n" + " but remap file is missing from soil erodibility parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + Field soil_erodibility( + FieldIdentifier(field_name, layout_2d, nondim, tgt_grid->name())); + soil_erodibility.allocate_view(); + + remapper->register_field_from_tgt(soil_erodibility); + + remapper->registration_ends(); + + return remapper; + +} // create_horiz_remapper + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- + +template +std::shared_ptr soilErodibilityFunctions::create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file) { + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(data_file, io_grid, io_fields, true); +} // create_data_reader + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- + +template +void soilErodibilityFunctions::update_soil_erodibility_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_1d &input) { + start_timer("EAMxx::soilErodibility::update_soil_erodibility_data_from_file"); + + // 1. Read from file + start_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_data"); + scorpio_reader->read_variables(); + stop_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_data"); + + // 2. Run the horiz remapper (it is a do-nothing op if soilErodibility data is on + // same grid as model) + start_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_remap"); + horiz_interp.remap(/*forward = */ true); + stop_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_remap"); + + // 3. Get the tgt field of the remapper + start_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_field"); + // Recall, the fields are registered in the order: + // Read the field from the file + input = horiz_interp.get_tgt_field(0).get_view(); + stop_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_field"); + + stop_timer("EAMxx::soilErodibility::update_soil_erodibility_data_from_file"); + +} // END update_soil_erodibility_data_from_file + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- + +template +void soilErodibilityFunctions::init_soil_erodibility_file_read( + const int ncol, const std::string field_name, const std::string dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &soilErodibilityHorizInterp, + std::shared_ptr &soilErodibilityDataReader) { + // Init horizontal remap + soilErodibilityHorizInterp = create_horiz_remapper( + grid, data_file, mapping_file, field_name, dim_name1); + + // Create reader (an AtmosphereInput object) + soilErodibilityDataReader = create_data_reader(soilErodibilityHorizInterp, data_file); +} // init_soil_erodibility_file_read +} // namespace soil_erodibility +} // namespace scream + +#endif // SOIL_ERODIBILITY_IMPL_HPP diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index b337ce3a503..8a109337eec 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -22,6 +22,8 @@ atmosphere_processes: srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + + soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc # FIXME: just set everything to a constant scalar for testing online_emis_IC_ncl_a1: 0.0 online_emis_IC_ncl_a2: 0.0 From 52f32c0015574e43572d19578e95ee5a3ed88a6d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 29 Oct 2024 12:13:54 -0700 Subject: [PATCH 331/366] Changes needed to start validatioon with EAM --- ...and_online_emissions_process_interface.cpp | 31 ++++++++++++++++++- .../mam/emissions/CMakeLists.txt | 3 +- .../single-process/mam/emissions/input.yaml | 8 +++-- externals/mam4xx | 2 +- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f6ce5dc9f82..5a327f4b1d4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -6,14 +6,17 @@ namespace scream { +// For reading soil erodibility file using soilErodibilityFunc = soil_erodibility::soilErodibilityFunctions; + // ================================================================ // Constructor // ================================================================ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { + // FIXME: Do we want to read dust emiss factor hfrom namelist?? /* Anything that can be initialized without grid information can be * initialized here. Like universal constants. */ @@ -47,6 +50,16 @@ void MAMSrfOnlineEmiss::set_grids( // For components of dust flux const FieldLayout vector4d = grid_->get_2d_vector_layout(4); + // FIXME: The following variables are used for validation ONLY!!! REMOVE + // THEM!!! + add_field("dstflx1", scalar2d, kg / m2 / s, grid_name); + add_field("dstflx2", scalar2d, kg / m2 / s, grid_name); + add_field("dstflx3", scalar2d, kg / m2 / s, grid_name); + add_field("dstflx4", scalar2d, kg / m2 / s, grid_name); + add_field("z_mid", scalar3d_m, kg / m2 / s, grid_name); + add_field("u_wind", scalar3d_m, kg / m2 / s, grid_name); + add_field("v_wind", scalar3d_m, kg / m2 / s, grid_name); + // -------------------------------------------------------------------------- // These variables are "Required" or pure inputs for the process // -------------------------------------------------------------------------- @@ -393,6 +406,20 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Online emissions from dust and sea salt //-------------------------------------------------------------------- + // FIXME: Remove the following vars as they are used only for validation + const const_view_2d z_mid = get_field_in("z_mid").get_view(); + const const_view_2d u_wind = get_field_in("u_wind").get_view(); + const const_view_2d v_wind = get_field_in("v_wind").get_view(); + const const_view_1d dstflx1 = + get_field_in("dstflx1").get_view(); + const const_view_1d dstflx2 = + get_field_in("dstflx2").get_view(); + const const_view_1d dstflx3 = + get_field_in("dstflx3").get_view(); + const const_view_1d dstflx4 = + get_field_in("dstflx4").get_view(); + // FIXME: Remove ^^^^^^ + const const_view_2d dstflx = get_field_in("dstflx").get_view(); auto constituent_fluxes = this->constituent_fluxes_; auto soil_erodibility = this->soil_erodibility_; @@ -400,11 +427,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // copy current values to online-emissions-local version // TODO: potentially combine with below parfor(icol) loop? Kokkos::parallel_for( - "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + // "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + "online_emis_fluxes", 1, KOKKOS_LAMBDA(int icol) { view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); mam4::aero_model_emissions::aero_model_emissions( dstflx, soil_erodibility, fluxes_col); }); + // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes // in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert // Kokkos::deep_copy(constituent_fluxes_, online_data.cfluxes); diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index c8e690b929f..8191338fb66 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -13,7 +13,8 @@ CreateADUnitTest(${TEST_BASE_NAME} # Set AD configurable options set (ATM_TIME_STEP 1800) -SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +#SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +SetVarDependingOnTestSize(NUM_STEPS 1 1 1) set (RUN_T0 2021-10-12-45000) ## Copy (and configure) yaml files needed by tests diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 8a109337eec..48a658ab151 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -64,17 +64,19 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + #Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + Filename: /qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/4src/screami_unit_tests_mam4xx_ne2np4L72_LAT_81p6352444000824_LON_44p7676953136545_ONLINE_EMISS_1st.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file #we should get the following variables from other processes pbl_height : 1.0 - dstflx : [ -0.99391637990504811E-011, -0.53350997938896468E-010, -0.12510222703847123E-009, -0.11784238261270793E-009] + dstflx : [-0.21955382571753977E-009, -0.11785111846584436E-008, -0.27634792128795150E-008, -0.26031109314006694E-008] ocnfrc: 0.99999999999999989E+000 - sst: 0.30196620710190155E+003 + sst: 0.27286547802237800E+003 cldfrac_tot: 0.138584624960092 + horiz_winds: [-0.63549092155149021E+001, 0.10220040376206645E+001] # The parameters for I/O control Scorpio: diff --git a/externals/mam4xx b/externals/mam4xx index c837e32f0f3..16f1ab9d97c 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit c837e32f0f3ee63d2a7b50bd5bcf9e9b29845265 +Subproject commit 16f1ab9d97c10faa0bbd50755b7d1e08506aa01a From dd258dff26f9e213ea79fd9de3fcef609e78fbd3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 29 Oct 2024 18:13:25 -0700 Subject: [PATCH 332/366] EAMxx: dust emissions matches EAMvs grdi cell --- ...and_online_emissions_process_interface.cpp | 45 +++++++++++++++++-- .../single-process/mam/emissions/input.yaml | 10 ++--- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 5a327f4b1d4..c82665361b1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -115,10 +115,14 @@ void MAMSrfOnlineEmiss::set_grids( // U and V components of the wind[m/s] add_field("horiz_winds", vector3d, m / s, grid_name); + //----------- Variables from coupler (ocean component)--------- + // Ocean fraction [unitless] + add_field("ocnfrac", scalar2d, nondim, grid_name); + // Sea surface temperature [K] add_field("sst", scalar2d, K, grid_name); - // dust fluxes [kg/m^2/s]: Four flux values for eacch column + // dust fluxes [kg/m^2/s]: Four flux values for each column add_field("dstflx", vector4d, kg / m2 / s, grid_name); // ------------------------------------------------------------- @@ -420,18 +424,51 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { get_field_in("dstflx4").get_view(); // FIXME: Remove ^^^^^^ + // dust fluxes [kg/m^2/s]: Four flux values for each column const const_view_2d dstflx = get_field_in("dstflx").get_view(); - auto constituent_fluxes = this->constituent_fluxes_; - auto soil_erodibility = this->soil_erodibility_; + + // Ocean fraction [unitless] + const const_view_1d ocnfrac = + get_field_in("ocnfrac").get_view(); + + // Sea surface temperature [K] + const const_view_1d sst = get_field_in("sst").get_view(); + + // U wind component [m/s] + const const_view_2d u_bottom = + get_field_in("horiz_winds").get_component(0).get_view(); + + // V wind component [m/s] + const const_view_2d v_bottom = + get_field_in("horiz_winds").get_component(1).get_view(); + + // Constituent fluxes [kg/m^2/s] + auto constituent_fluxes = this->constituent_fluxes_; + + // Soil edodibility [fraction] + auto soil_erodibility = this->soil_erodibility_; + // TODO: check that units are consistent with srf emissions! // copy current values to online-emissions-local version // TODO: potentially combine with below parfor(icol) loop? + const int surf_lev = nlev_ - 1; + Kokkos::parallel_for( // "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { "online_emis_fluxes", 1, KOKKOS_LAMBDA(int icol) { + // gather all input + const const_view_1d dstflx_icol = ekat::subview(dstflx, icol); + const const_view_1d u_bottom_icol = ekat::subview(u_bottom, icol); + const const_view_1d v_bottom_icol = ekat::subview(v_bottom, icol); + + // output view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); + + // comput online emissions mam4::aero_model_emissions::aero_model_emissions( - dstflx, soil_erodibility, fluxes_col); + sst(icol), ocnfrac(icol), u_bottom(icol, surf_lev), + v_bottom(icol, surf_lev), dstflx_icol, soil_erodibility(icol), + fluxes_col); }); // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 48a658ab151..3f41c3d9180 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -65,18 +65,18 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. #Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} - Filename: /qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/4src/screami_unit_tests_mam4xx_ne2np4L72_LAT_81p6352444000824_LON_44p7676953136545_ONLINE_EMISS_1st.nc + Filename: /qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/4src/screami_unit_tests_mam4xx_ne2np4L72_LAT_39p0553303768561_LON_83p8339931334693_ONLINE_EMISS_1st.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file #we should get the following variables from other processes pbl_height : 1.0 - dstflx : [-0.21955382571753977E-009, -0.11785111846584436E-008, -0.27634792128795150E-008, -0.26031109314006694E-008] - ocnfrc: 0.99999999999999989E+000 - sst: 0.27286547802237800E+003 + dstflx : [ -0.15639291491198312E-010, -0.83947887868839890E-010, -0.19684857140996447E-009, -0.18542519360366344E-009] + ocnfrac: 0.0 + sst: 0.0 cldfrac_tot: 0.138584624960092 - horiz_winds: [-0.63549092155149021E+001, 0.10220040376206645E+001] + horiz_winds: [0.70975328186226694E+001, 0.83157735469095362E+000] # The parameters for I/O control Scorpio: From 9e0ad96960f1fe94a944d61e0d7e57c03ca29028 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 29 Oct 2024 19:09:14 -0700 Subject: [PATCH 333/366] EAMxx:calculate_seasalt_numflux_in_bins validated --- ...and_online_emissions_process_interface.cpp | 30 ++++++++----------- externals/mam4xx | 2 +- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index c82665361b1..ff239861dd4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -57,9 +57,6 @@ void MAMSrfOnlineEmiss::set_grids( add_field("dstflx3", scalar2d, kg / m2 / s, grid_name); add_field("dstflx4", scalar2d, kg / m2 / s, grid_name); add_field("z_mid", scalar3d_m, kg / m2 / s, grid_name); - add_field("u_wind", scalar3d_m, kg / m2 / s, grid_name); - add_field("v_wind", scalar3d_m, kg / m2 / s, grid_name); - // -------------------------------------------------------------------------- // These variables are "Required" or pure inputs for the process // -------------------------------------------------------------------------- @@ -411,9 +408,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- // FIXME: Remove the following vars as they are used only for validation - const const_view_2d z_mid = get_field_in("z_mid").get_view(); - const const_view_2d u_wind = get_field_in("u_wind").get_view(); - const const_view_2d v_wind = get_field_in("v_wind").get_view(); + const const_view_2d z_mid2 = get_field_in("z_mid").get_view(); const const_view_1d dstflx1 = get_field_in("dstflx1").get_view(); const const_view_1d dstflx2 = @@ -435,11 +430,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { const const_view_1d sst = get_field_in("sst").get_view(); // U wind component [m/s] - const const_view_2d u_bottom = + const const_view_2d u_wind = get_field_in("horiz_winds").get_component(0).get_view(); // V wind component [m/s] - const const_view_2d v_bottom = + const const_view_2d v_wind = get_field_in("horiz_winds").get_component(1).get_view(); // Constituent fluxes [kg/m^2/s] @@ -448,6 +443,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Soil edodibility [fraction] auto soil_erodibility = this->soil_erodibility_; + // Vertical layer height at midpoints + const const_view_2d z_mid = dry_atm_.z_mid; + // TODO: check that units are consistent with srf emissions! // copy current values to online-emissions-local version // TODO: potentially combine with below parfor(icol) loop? @@ -456,19 +454,17 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::parallel_for( // "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { "online_emis_fluxes", 1, KOKKOS_LAMBDA(int icol) { - // gather all input - const const_view_1d dstflx_icol = ekat::subview(dstflx, icol); - const const_view_1d u_bottom_icol = ekat::subview(u_bottom, icol); - const const_view_1d v_bottom_icol = ekat::subview(v_bottom, icol); + // input + const const_view_1d dstflx_icol = ekat::subview(dstflx, icol); // output view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); // comput online emissions mam4::aero_model_emissions::aero_model_emissions( - sst(icol), ocnfrac(icol), u_bottom(icol, surf_lev), - v_bottom(icol, surf_lev), dstflx_icol, soil_erodibility(icol), - fluxes_col); + sst(icol), ocnfrac(icol), u_wind(icol, surf_lev), + v_wind(icol, surf_lev), z_mid(icol, surf_lev), dstflx_icol, + soil_erodibility(icol), fluxes_col); }); // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes @@ -525,8 +521,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // the structs.{values} holding the above are: // SeasaltEmissionsData.{mpoly, mprot, mlip} // DustEmissionsData.dust_dmt_vwr -// OnlineEmissionsData.{dust_flux_in, surface_temp, u_bottom, -// v_bottom, z_bottom, ocean_frac} +// OnlineEmissionsData.{dust_flux_in, surface_temp, u_wind, +// v_wind, z_bottom, ocean_frac} // NOTE: fortran mam4 gets dust_flux_in and ocean_frac from cam_in, // which is a chunk-wise variable at this point // (see: physpkg.F90:{1605 & 1327}) otherwise grabs the end/bottom diff --git a/externals/mam4xx b/externals/mam4xx index 16f1ab9d97c..328abdcc7fb 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 16f1ab9d97c10faa0bbd50755b7d1e08506aa01a +Subproject commit 328abdcc7fba6ca9f122cb204da2773b1e7792cf From a7d47e949f778efd34e28f2650af84a97c0563b9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 30 Oct 2024 11:17:01 -0700 Subject: [PATCH 334/366] EAMxx: marine_organic_emis validated with hardwired file read mom --- ...and_online_emissions_process_interface.cpp | 39 +++++++++++-------- .../single-process/mam/emissions/input.yaml | 10 ++--- externals/mam4xx | 2 +- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index ff239861dd4..75524dbdc53 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -52,10 +52,6 @@ void MAMSrfOnlineEmiss::set_grids( // FIXME: The following variables are used for validation ONLY!!! REMOVE // THEM!!! - add_field("dstflx1", scalar2d, kg / m2 / s, grid_name); - add_field("dstflx2", scalar2d, kg / m2 / s, grid_name); - add_field("dstflx3", scalar2d, kg / m2 / s, grid_name); - add_field("dstflx4", scalar2d, kg / m2 / s, grid_name); add_field("z_mid", scalar3d_m, kg / m2 / s, grid_name); // -------------------------------------------------------------------------- // These variables are "Required" or pure inputs for the process @@ -252,18 +248,37 @@ void MAMSrfOnlineEmiss::set_grids( const std::string soil_erodibility_data_file = m_params.get("soil_erodibility_file"); - // /compyfs/inputdata/atm/cam/dst/dst_1.9x2.5_c090203.nc // Field to be read from file - const std::string field_name = "mbl_bsn_fct_geo"; + const std::string soil_erod_fld_name = "mbl_bsn_fct_geo"; // Dimensions of the filed - const std::string dim_name1 = "ncol"; + const std::string soil_erod_dname = "ncol"; // initialize the file read soilErodibilityFunc::init_soil_erodibility_file_read( - ncol_, field_name, dim_name1, grid_, soil_erodibility_data_file, + ncol_, soil_erod_fld_name, soil_erod_dname, grid_, + soil_erodibility_data_file, srf_map_file, horizInterp_, + dataReader_); // output + // ------------------------------------------------------------- + // setup to enable reading marine organics file + // ------------------------------------------------------------- +#if 0 + const std::string marine_organics_data_file = + m_params.get("marine_organics_file"); + + // Field to be read from file + const std::vector marine_org_fld_name = { + "CHL1", "TRUEPOLYC", "TRUEPROTC", "TRUELIPC"}; + + // Dimensions of the filed + const std::string marine_org_dname = "ncol"; + + // initialize the file read + soilErodibilityFunc::init_soil_erodibility_file_read( + ncol_, marine_org_fld_name, marine_org_dname, grid_, marine_organics_data_file, srf_map_file, horizInterp_, dataReader_); // output +#endif } // set_grid ends @@ -409,14 +424,6 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // FIXME: Remove the following vars as they are used only for validation const const_view_2d z_mid2 = get_field_in("z_mid").get_view(); - const const_view_1d dstflx1 = - get_field_in("dstflx1").get_view(); - const const_view_1d dstflx2 = - get_field_in("dstflx2").get_view(); - const const_view_1d dstflx3 = - get_field_in("dstflx3").get_view(); - const const_view_1d dstflx4 = - get_field_in("dstflx4").get_view(); // FIXME: Remove ^^^^^^ // dust fluxes [kg/m^2/s]: Four flux values for each column diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 3f41c3d9180..b8b97e11185 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -65,18 +65,18 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. #Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} - Filename: /qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/4src/screami_unit_tests_mam4xx_ne2np4L72_LAT_39p0553303768561_LON_83p8339931334693_ONLINE_EMISS_1st.nc + Filename: /qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/4src/screami_unit_tests_mam4xx_ne2np4L72_LAT_m5p57727434969464_LON_83p9312243922771_ONLINE_EMISS_1st.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file #we should get the following variables from other processes pbl_height : 1.0 - dstflx : [ -0.15639291491198312E-010, -0.83947887868839890E-010, -0.19684857140996447E-009, -0.18542519360366344E-009] - ocnfrac: 0.0 - sst: 0.0 + dstflx : [ 0.00000000000000000E+000, 0.00000000000000000E+000, 0.00000000000000000E+000, 0.00000000000000000E+000] + ocnfrac: 0.10000000000000000E+001 + sst: 0.30178553874977507E+003 cldfrac_tot: 0.138584624960092 - horiz_winds: [0.70975328186226694E+001, 0.83157735469095362E+000] + horiz_winds: [-0.24988988196194634E+000, -0.23959782871450760E+000] # The parameters for I/O control Scorpio: diff --git a/externals/mam4xx b/externals/mam4xx index 328abdcc7fb..4f6259f61ae 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 328abdcc7fba6ca9f122cb204da2773b1e7792cf +Subproject commit 4f6259f61aeda3814df542e541a7a48de35e255e From 059466761743ff86359241a83d02da13897e7c49 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 30 Oct 2024 17:05:37 -0700 Subject: [PATCH 335/366] EAMxx: Partial implementation of marine orgaics file read --- .../cime_config/namelist_defaults_scream.xml | 16 +- ...and_online_emissions_process_interface.cpp | 21 ++- ...and_online_emissions_process_interface.hpp | 17 +- .../physics/mam/readfiles/marine_organics.hpp | 111 ++++++++++++ .../mam/readfiles/marine_organics_impl.hpp | 169 ++++++++++++++++++ .../mam/readfiles/soil_erodibility.hpp | 2 +- .../mam/readfiles/soil_erodibility_impl.hpp | 34 ++-- .../mam/emissions/CMakeLists.txt | 2 + .../single-process/mam/emissions/input.yaml | 1 + 9 files changed, 338 insertions(+), 35 deletions(-) create mode 100644 components/eamxx/src/physics/mam/readfiles/marine_organics.hpp create mode 100644 components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 40211361376..8b3a1131d2a 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -382,16 +382,10 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/dst_ne4pg2_c20241028.nc >>>>>>> EAMxx:Adds soil erodibility file read and sent to online emission read - - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne30pg2_c20241030.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne4pg2_c20241030.nc @@ -618,6 +612,7 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_CA_ne32_x32_v1_pg2_16xdel2.nc + ${CASE}.scream 0.0 0.0 @@ -703,6 +698,7 @@ be lost if SCREAM_HACK_XML is not enabled. + ./${CASE}.scream default ${REST_N} diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 75524dbdc53..a7514c7ee5d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -258,27 +258,30 @@ void MAMSrfOnlineEmiss::set_grids( // initialize the file read soilErodibilityFunc::init_soil_erodibility_file_read( ncol_, soil_erod_fld_name, soil_erod_dname, grid_, - soil_erodibility_data_file, srf_map_file, horizInterp_, - dataReader_); // output + soil_erodibility_data_file, srf_map_file, serod_horizInterp_, + serod_dataReader_); // output // ------------------------------------------------------------- // setup to enable reading marine organics file // ------------------------------------------------------------- -#if 0 + const std::string marine_organics_data_file = m_params.get("marine_organics_file"); + // /compyfs/inputdata/atm/cam/chem/trop_mam/marine_BGC/monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc // Field to be read from file const std::vector marine_org_fld_name = { - "CHL1", "TRUEPOLYC", "TRUEPROTC", "TRUELIPC"}; + "TRUEPOLYC", "TRUEPROTC", "TRUELIPC"}; // Dimensions of the filed const std::string marine_org_dname = "ncol"; // initialize the file read - soilErodibilityFunc::init_soil_erodibility_file_read( - ncol_, marine_org_fld_name, marine_org_dname, grid_, marine_organics_data_file, - srf_map_file, horizInterp_, dataReader_); // output -#endif + marineOrganicsFunc::init_marine_organics_file_read( + ncol_, marine_org_fld_name, marine_org_dname, grid_, + marine_organics_data_file, srf_map_file, + // output + morg_horizInterp_, morg_data_start_, morg_data_end_, morg_data_out_, + morg_dataReader_); } // set_grid ends @@ -385,7 +388,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // This data is time-independent, we read all data here for the // entire simulation soilErodibilityFunc::update_soil_erodibility_data_from_file( - dataReader_, *horizInterp_, + serod_dataReader_, *serod_horizInterp_, soil_erodibility_); // output //-------------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index c1915cd81ba..fb162ee94e4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -9,6 +9,9 @@ #include #include +// For reading marine organics file +#include + // For declaring surface and online emission class derived from atm process // class #include @@ -54,13 +57,16 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu // For reading soil erodibility file - std::shared_ptr horizInterp_; - std::shared_ptr dataReader_; + std::shared_ptr serod_horizInterp_; + std::shared_ptr serod_dataReader_; const_view_1d soil_erodibility_; public: using srfEmissFunc = mam_coupling::srfEmissFunctions; using onlineEmiss = mam_coupling::onlineEmissions; + // For reading marine organics file + using marineOrganicsFunc = + marine_organics::marineOrganicsFunctions; // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -168,6 +174,13 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { onlineEmiss online_emissions; + // For reading marine organics file + std::shared_ptr morg_horizInterp_; + std::shared_ptr morg_dataReader_; + marineOrganicsFunc::marineOrganicsTimeState morg_timeState_; + marineOrganicsFunc::marineOrganicsInput morg_data_start_, morg_data_end_; + marineOrganicsFunc::marineOrganicsOutput morg_data_out_; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp new file mode 100644 index 00000000000..0c472fdb44c --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp @@ -0,0 +1,111 @@ +#ifndef MARINE_ORGANICS_HPP +#define MARINE_ORGANICS_HPP + +// For AtmosphereInput +#include "share/io/scorpio_input.hpp" + +namespace scream { +namespace marine_organics { + +template +struct marineOrganicsFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using view_2d = typename KT::template view_2d; + + struct marineOrganicsTimeState { + marineOrganicsTimeState() = default; + // Whether the timestate has been initialized. + // The current month + int current_month = -1; + // Julian Date for the beginning of the month, as defined in + // /src/share/util/scream_time_stamp.hpp + // See this file for definition of Julian Date. + Real t_beg_month; + // Current simulation Julian Date + Real t_now; + // Number of days in the current month, cast as a Real + Real days_this_month; + }; // marineOrganicsTimeState + + struct marineOrganicsData { + marineOrganicsData() = default; + marineOrganicsData(const int ncol_, const int nsectors_) + : ncols(ncol_), nsectors(nsectors_) { + init(ncols, nsectors, true); + } + + void init(const int ncol, const int nsector, const bool allocate) { + ncols = ncol; + nsectors = nsector; + if(allocate) emiss_sectors = view_2d("morgAllSectors", nsectors, ncols); + } // marineOrganicsData init + + // Basic spatial dimensions of the data + int ncols, nsectors; + view_2d emiss_sectors; + }; // marineOrganicsData + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + struct marineOrganicsInput { + marineOrganicsInput() = default; + marineOrganicsInput(const int ncols_, const int nsectors_) { + init(ncols_, nsectors_); + } + + void init(const int ncols_, const int nsectors_) { + data.init(ncols_, nsectors_, true); + } + marineOrganicsData data; // All marineOrganics fields + }; // marineOrganicsInput + + // The output is really just marineOrganicsData, but for clarity it might + // help to see a marineOrganicsOutput along a marineOrganicsInput in functions + // signatures + using marineOrganicsOutput = marineOrganicsData; + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &marineOrganics_data_file, const std::string &map_file, + const std::vector &field_name, const std::string &dim_name1); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + static std::shared_ptr create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- +#if 0 + static void update_marine_organics_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_1d &input); +#endif + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + + static void init_marine_organics_file_read( + const int ncol, const std::vector field_name, + const std::string dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &marineOrganicsHorizInterp, + marineOrganicsInput morg_data_start_, marineOrganicsInput morg_data_end_, + marineOrganicsData morg_data_out_, + std::shared_ptr &marineOrganicsDataReader); + +}; // struct marineOrganicsFunctions + +} // namespace marine_organics +} // namespace scream +#endif // MARINE_ORGANICS_HPP + +#include "marine_organics_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp new file mode 100644 index 00000000000..ba97affafd5 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp @@ -0,0 +1,169 @@ +#ifndef MARINE_ORGANICS_IMPL_HPP +#define MARINE_ORGANICS_IMPL_HPP + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" + +namespace scream { +namespace marine_organics { + +template +std::shared_ptr +marineOrganicsFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const std::vector &field_name, const std::string &dim_name1) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, dim_name1); + + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("marine_organics_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen marine organics " + "data to fit the model. We only allow\n" + " marine organics data to be at the same or " + "coarser resolution as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG(map_file != "", + "ERROR: marine organics data is on a different grid " + "than the model one,\n" + " but remap file is missing from marine organics " + "parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + std::vector fields_vector; + + const int field_size = field_name.size(); + for(int icomp = 0; icomp < field_size; ++icomp) { + auto comp_name = field_name[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); + f.allocate_view(); + fields_vector.push_back(f); + remapper->register_field_from_tgt(f); + } + + remapper->registration_ends(); + + return remapper; + +} // create_horiz_remapper + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- + +template +std::shared_ptr +marineOrganicsFunctions::create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file) { + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(data_file, io_grid, io_fields, true); +} // create_data_reader + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- +#if 0 +template +void marineOrganicsFunctions::update_marine_organics_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_1d &input) { + start_timer("EAMxx::marineOrganics::update_marine_organics_data_from_file"); + + // 1. Read from file + start_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::read_" + "data"); + scorpio_reader->read_variables(); + stop_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::read_" + "data"); + + // 2. Run the horiz remapper (it is a do-nothing op if marineOrganics data is + // on same grid as model) + start_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::horiz_" + "remap"); + horiz_interp.remap(/*forward = */ true); + stop_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::horiz_" + "remap"); + + // 3. Get the tgt field of the remapper + start_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::get_" + "field"); + // Recall, the fields are registered in the order: + // Read the field from the file + input = horiz_interp.get_tgt_field(0).get_view(); + stop_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::get_" + "field"); + + stop_timer("EAMxx::marineOrganics::update_marine_organics_data_from_file"); + +} // END update_marine_organics_data_from_file +#endif +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- + +template +void marineOrganicsFunctions::init_marine_organics_file_read( + const int ncol, const std::vector field_name, + const std::string dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &marineOrganicsHorizInterp, + marineOrganicsInput data_start_, marineOrganicsInput data_end_, + marineOrganicsData data_out_, + std::shared_ptr &marineOrganicsDataReader) { + // Init horizontal remap + + marineOrganicsHorizInterp = create_horiz_remapper( + grid, data_file, mapping_file, field_name, dim_name1); + + // Initialize the size of start/end/out data structures + data_start_ = marineOrganicsInput(ncol, field_name.size()); + data_end_ = marineOrganicsInput(ncol, field_name.size()); + data_out_.init(ncol, 1, true); + + // Create reader (an AtmosphereInput object) + marineOrganicsDataReader = + create_data_reader(marineOrganicsHorizInterp, data_file); + +} // init_marine_organics_file_read +} // namespace marine_organics +} // namespace scream + +#endif // MARINE_ORGANICS_IMPL_HPP \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp index 9be7c0f817b..223fb78cfc1 100644 --- a/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp @@ -50,7 +50,7 @@ struct soilErodibilityFunctions { }; // struct soilErodilityFunctions -} // namespace soil_erodibilty +} // namespace soil_erodibility } // namespace scream #endif // SOIL_ERODIBILITY_HPP diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp index 2f6a5da6888..a685134419f 100644 --- a/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp @@ -15,11 +15,10 @@ soilErodibilityFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &data_file, const std::string &map_file, const std::string &field_name, const std::string &dim_name1) { - using namespace ShortFieldTagsNames; scorpio::register_file(data_file, scorpio::Read); - const int ncols_data = scorpio::get_dimlen(data_file, dim_name1); + const int ncols_data = scorpio::get_dimlen(data_file, dim_name1); scorpio::release_file(data_file); @@ -44,7 +43,8 @@ soilErodibilityFunctions::create_horiz_remapper( EKAT_REQUIRE_MSG(map_file != "", "ERROR: soil erodibility data is on a different grid " "than the model one,\n" - " but remap file is missing from soil erodibility parameter list."); + " but remap file is missing from soil erodibility " + "parameter list."); remapper = std::make_shared(horiz_interp_tgt_grid, map_file); @@ -73,7 +73,8 @@ soilErodibilityFunctions::create_horiz_remapper( // ------------------------------------------------------------------------------------------- template -std::shared_ptr soilErodibilityFunctions::create_data_reader( +std::shared_ptr +soilErodibilityFunctions::create_data_reader( const std::shared_ptr &horiz_remapper, const std::string &data_file) { std::vector io_fields; @@ -95,27 +96,33 @@ void soilErodibilityFunctions::update_soil_erodibility_data_from_file( // 1. Read from file start_timer( - "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_data"); + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_" + "data"); scorpio_reader->read_variables(); stop_timer( - "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_data"); + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_" + "data"); - // 2. Run the horiz remapper (it is a do-nothing op if soilErodibility data is on - // same grid as model) + // 2. Run the horiz remapper (it is a do-nothing op if soilErodibility data is + // on same grid as model) start_timer( - "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_remap"); + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_" + "remap"); horiz_interp.remap(/*forward = */ true); stop_timer( - "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_remap"); + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_" + "remap"); // 3. Get the tgt field of the remapper start_timer( - "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_field"); + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_" + "field"); // Recall, the fields are registered in the order: // Read the field from the file input = horiz_interp.get_tgt_field(0).get_view(); stop_timer( - "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_field"); + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_" + "field"); stop_timer("EAMxx::soilErodibility::update_soil_erodibility_data_from_file"); @@ -137,7 +144,8 @@ void soilErodibilityFunctions::init_soil_erodibility_file_read( grid, data_file, mapping_file, field_name, dim_name1); // Create reader (an AtmosphereInput object) - soilErodibilityDataReader = create_data_reader(soilErodibilityHorizInterp, data_file); + soilErodibilityDataReader = + create_data_reader(soilErodibilityHorizInterp, data_file); } // init_soil_erodibility_file_read } // namespace soil_erodibility } // namespace scream diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index 8191338fb66..4b7350a1bb9 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -37,6 +37,8 @@ set (TEST_INPUT_FILES scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc + scream/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index b8b97e11185..ce103765b70 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -24,6 +24,7 @@ atmosphere_processes: srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc + marine_organics_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc # FIXME: just set everything to a constant scalar for testing online_emis_IC_ncl_a1: 0.0 online_emis_IC_ncl_a2: 0.0 From 2f0d6c8c84d913556b9277dab038e154c1d79bbb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 30 Oct 2024 20:10:28 -0700 Subject: [PATCH 336/366] EAMxx:Reading and interpolation of mariane organic file works, not tested yet --- ...and_online_emissions_process_interface.cpp | 31 +++- .../physics/mam/readfiles/marine_organics.hpp | 68 +++++--- .../mam/readfiles/marine_organics_impl.hpp | 150 ++++++++++++++++-- 3 files changed, 217 insertions(+), 32 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index a7514c7ee5d..cdabc48a7d9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -260,10 +260,10 @@ void MAMSrfOnlineEmiss::set_grids( ncol_, soil_erod_fld_name, soil_erod_dname, grid_, soil_erodibility_data_file, srf_map_file, serod_horizInterp_, serod_dataReader_); // output + // ------------------------------------------------------------- // setup to enable reading marine organics file // ------------------------------------------------------------- - const std::string marine_organics_data_file = m_params.get("marine_organics_file"); // /compyfs/inputdata/atm/cam/chem/trop_mam/marine_BGC/monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc @@ -282,6 +282,7 @@ void MAMSrfOnlineEmiss::set_grids( // output morg_horizInterp_, morg_data_start_, morg_data_end_, morg_data_out_, morg_dataReader_); + printf("END SIZE:%i,%i\n", morg_data_end_.data.ncols, ncol_); } // set_grid ends @@ -391,6 +392,14 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { serod_dataReader_, *serod_horizInterp_, soil_erodibility_); // output + //-------------------------------------------------------------------- + // Update marine orgaincs from file + //-------------------------------------------------------------------- + + marineOrganicsFunc::update_marine_organics_data_from_file( + morg_dataReader_, timestamp(), curr_month, *morg_horizInterp_, + morg_data_end_); // output + //-------------------------------------------------------------------- // Initialize online emissions from file //-------------------------------------------------------------------- @@ -421,10 +430,28 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // ALSO this code should in the preprocessor struct now Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); + // Gather time and state information for interpolation + const auto ts = timestamp() + dt; + //-------------------------------------------------------------------- // Online emissions from dust and sea salt //-------------------------------------------------------------------- + // --- Interpolate mariane organics data -- + + // Update TimeState, note the addition of dt + morg_timeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + marineOrganicsFunc::update_marine_organics_timestate( + morg_dataReader_, ts, *morg_horizInterp_, + // output + morg_timeState_, morg_data_start_, morg_data_end_); + + // Call the main marineOrganics routine to get interpolated aerosol forcings. + marineOrganicsFunc::marineOrganics_main(morg_timeState_, morg_data_start_, + morg_data_end_, morg_data_out_); + // FIXME: Remove the following vars as they are used only for validation const const_view_2d z_mid2 = get_field_in("z_mid").get_view(); // FIXME: Remove ^^^^^^ @@ -486,7 +513,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Interpolate srf emiss data read in from emissions files //-------------------------------------------------------------------- // Gather time and state information for interpolation - auto ts = timestamp() + dt; + // auto ts = timestamp() + dt; for(srf_emiss_ &ispec_srf : srf_emiss_species_) { // Update TimeState, note the addition of dt diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp index 0c472fdb44c..92136d315f5 100644 --- a/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp @@ -11,9 +11,12 @@ template struct marineOrganicsFunctions { using Device = DeviceType; - using KT = KokkosTypes; - using view_2d = typename KT::template view_2d; + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + using view_2d = typename KT::template view_2d; + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- struct marineOrganicsTimeState { marineOrganicsTimeState() = default; // Whether the timestate has been initialized. @@ -31,12 +34,12 @@ struct marineOrganicsFunctions { struct marineOrganicsData { marineOrganicsData() = default; - marineOrganicsData(const int ncol_, const int nsectors_) - : ncols(ncol_), nsectors(nsectors_) { + marineOrganicsData(const int &ncol_, const int &nfields_) + : ncols(ncol_), nsectors(nfields_) { init(ncols, nsectors, true); } - void init(const int ncol, const int nsector, const bool allocate) { + void init(const int &ncol, const int &nsector, const bool allocate) { ncols = ncol; nsectors = nsector; if(allocate) emiss_sectors = view_2d("morgAllSectors", nsectors, ncols); @@ -51,12 +54,12 @@ struct marineOrganicsFunctions { // ------------------------------------------------------------------------------------------- struct marineOrganicsInput { marineOrganicsInput() = default; - marineOrganicsInput(const int ncols_, const int nsectors_) { - init(ncols_, nsectors_); + marineOrganicsInput(const int &ncols_, const int &nfields_) { + init(ncols_, nfields_); } - void init(const int ncols_, const int nsectors_) { - data.init(ncols_, nsectors_, true); + void init(const int &ncols_, const int &nfields_) { + data.init(ncols_, nfields_, true); } marineOrganicsData data; // All marineOrganics fields }; // marineOrganicsInput @@ -68,7 +71,6 @@ struct marineOrganicsFunctions { // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- - static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &marineOrganics_data_file, const std::string &map_file, @@ -76,30 +78,60 @@ struct marineOrganicsFunctions { // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- - static std::shared_ptr create_data_reader( const std::shared_ptr &horiz_remapper, const std::string &data_file); // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- -#if 0 static void update_marine_organics_data_from_file( std::shared_ptr &scorpio_reader, - AbstractRemapper &horiz_interp, const_view_1d &input); -#endif + const util::TimeStamp &ts, + const int &time_index, // zero-based + AbstractRemapper &horiz_interp, + marineOrganicsInput &marineOrganics_input); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + static void update_marine_organics_timestate( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, AbstractRemapper &horiz_interp, + marineOrganicsTimeState &time_state, marineOrganicsInput &beg, + marineOrganicsInput &end); + // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- + static void marineOrganics_main(const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, + const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out); + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + static void perform_time_interpolation( + const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- + // Performs convex interpolation of x0 and x1 at point t + template + KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t); + + // ------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------------------------------- static void init_marine_organics_file_read( - const int ncol, const std::vector field_name, - const std::string dim_name1, + const int &ncol, const std::vector &field_name, + const std::string &dim_name1, const std::shared_ptr &grid, const std::string &data_file, const std::string &mapping_file, // output std::shared_ptr &marineOrganicsHorizInterp, - marineOrganicsInput morg_data_start_, marineOrganicsInput morg_data_end_, - marineOrganicsData morg_data_out_, + marineOrganicsInput &morg_data_start_, + marineOrganicsInput &morg_data_end_, marineOrganicsData &morg_data_out_, std::shared_ptr &marineOrganicsDataReader); }; // struct marineOrganicsFunctions diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp index ba97affafd5..8f532d5342a 100644 --- a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp @@ -22,9 +22,8 @@ marineOrganicsFunctions::create_horiz_remapper( scorpio::release_file(data_file); - // We could use model_grid directly if using same num levels, - // but since shallow clones are cheap, we may as well do it (less lines of - // code) + // Since shallow clones are cheap, we may as well do it (less lines of + // code) auto horiz_interp_tgt_grid = model_grid->clone("marine_organics_horiz_interp_tgt_grid", true); @@ -93,11 +92,11 @@ marineOrganicsFunctions::create_data_reader( // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- -#if 0 template void marineOrganicsFunctions::update_marine_organics_data_from_file( - std::shared_ptr &scorpio_reader, - AbstractRemapper &horiz_interp, const_view_1d &input) { + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + const int &time_index, // zero-based + AbstractRemapper &horiz_interp, marineOrganicsInput &marineOrganics_input) { start_timer("EAMxx::marineOrganics::update_marine_organics_data_from_file"); // 1. Read from file @@ -125,7 +124,9 @@ void marineOrganicsFunctions::update_marine_organics_data_from_file( "field"); // Recall, the fields are registered in the order: // Read the field from the file +#if 0 input = horiz_interp.get_tgt_field(0).get_view(); +#endif stop_timer( "EAMxx::marineOrganics::update_marine_organics_data_from_file::get_" "field"); @@ -133,20 +134,145 @@ void marineOrganicsFunctions::update_marine_organics_data_from_file( stop_timer("EAMxx::marineOrganics::update_marine_organics_data_from_file"); } // END update_marine_organics_data_from_file -#endif + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::update_marine_organics_timestate( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + AbstractRemapper &horiz_interp, marineOrganicsTimeState &time_state, + marineOrganicsInput &beg, marineOrganicsInput &end) { + // Now we check if we have to update the data that changes monthly + // NOTE: This means that marineOrganics assumes monthly data to update. Not + // any other frequency. + const auto month = ts.get_month() - 1; // Make it 0-based + if(month != time_state.current_month) { + // Update the marineOrganics time state information + time_state.current_month = month; + time_state.t_beg_month = + util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) + .frac_of_year_in_days(); + time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + + // Copy end'data into beg'data, and read in the new + // end + std::swap(beg, end); + + // Update the marineOrganics forcing data for this month and next month + // Start by copying next months data to this months data structure. + // NOTE: If the timestep is bigger than monthly this could cause the wrong + // values + // to be assigned. A timestep greater than a month is very unlikely + // so we will proceed. + int next_month = (time_state.current_month + 1) % 12; + update_marine_organics_data_from_file(scorpio_reader, ts, next_month, + horiz_interp, end); + } + +} // END updata_marine_organics_timestate + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- +template +template +KOKKOS_INLINE_FUNCTION ScalarX marineOrganicsFunctions::linear_interp( + const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { + return (1 - t) * x0 + t * x1; +} // linear_interp + // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::perform_time_interpolation( + const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out) { + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + + // Gather time stamp info + auto &t_now = time_state.t_now; + auto &t_beg = time_state.t_beg_month; + auto &delta_t = time_state.days_this_month; + + // At this stage, begin/end must have the same dimensions + EKAT_REQUIRE(data_end.data.ncols == data_beg.data.ncols); + + auto delta_t_fraction = (t_now - t_beg) / delta_t; + + EKAT_REQUIRE_MSG(delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of " + "[0,1].\n t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n delta_t: " + std::to_string(delta_t) + "\n"); + const int nsectors = data_beg.data.nsectors; + const int ncols = data_beg.data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nsectors); + + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int icol = team.league_rank(); // column index + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, 0u, nsectors), [&](int isec) { + const auto beg = data_beg.data.emiss_sectors(isec, icol); + const auto end = data_end.data.emiss_sectors(isec, icol); + data_out.emiss_sectors(isec, icol) = + linear_interp(beg, end, delta_t_fraction); + }); + }); + Kokkos::fence(); + +} // perform_time_interpolation + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::marineOrganics_main( + const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out) { + // Beg/End/Tmp month must have all sizes matching + + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_beg.data.ncols, + "Error! marineOrganicsInput data structs must have the same number of " + "columns.\n"); + + // Horiz interpolation can be expensive, and does not depend on the particular + // time of the month, so it can be done ONCE per month, *outside* + // marineOrganics_main (when updating the beg/end states, reading them from + // file). + EKAT_REQUIRE_MSG(data_end.data.ncols == data_out.ncols, + "Error! Horizontal interpolation is performed *before* " + "calling marineOrganics_main,\n" + " marineOrganicsInput and marineOrganicsOutput data " + "structs must have the " + "same number columns " + << data_end.data.ncols << " " << data_out.ncols + << ".\n"); + + // Step 1. Perform time interpolation + perform_time_interpolation(time_state, data_beg, data_end, data_out); +} // marineOrganics_main + +// ------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------- template void marineOrganicsFunctions::init_marine_organics_file_read( - const int ncol, const std::vector field_name, - const std::string dim_name1, + const int &ncol, const std::vector &field_name, + const std::string &dim_name1, const std::shared_ptr &grid, const std::string &data_file, const std::string &mapping_file, // output std::shared_ptr &marineOrganicsHorizInterp, - marineOrganicsInput data_start_, marineOrganicsInput data_end_, - marineOrganicsData data_out_, + marineOrganicsInput &data_start_, marineOrganicsInput &data_end_, + marineOrganicsData &data_out_, std::shared_ptr &marineOrganicsDataReader) { // Init horizontal remap @@ -156,7 +282,7 @@ void marineOrganicsFunctions::init_marine_organics_file_read( // Initialize the size of start/end/out data structures data_start_ = marineOrganicsInput(ncol, field_name.size()); data_end_ = marineOrganicsInput(ncol, field_name.size()); - data_out_.init(ncol, 1, true); + data_out_.init(ncol, field_name.size(), true); // Create reader (an AtmosphereInput object) marineOrganicsDataReader = From 82e843b479ea22cae32755fb25c61aa1ce43c2ac Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 30 Oct 2024 20:42:03 -0700 Subject: [PATCH 337/366] EAMxx: Connects marine organics emiss with MAM4xx codes --- ...am_srf_and_online_emissions_process_interface.cpp | 12 +++++++++--- externals/mam4xx | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index cdabc48a7d9..4b3a7470d1f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -268,7 +268,7 @@ void MAMSrfOnlineEmiss::set_grids( m_params.get("marine_organics_file"); // /compyfs/inputdata/atm/cam/chem/trop_mam/marine_BGC/monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc - // Field to be read from file + // Field to be read from file (order matters as they are read in same order) const std::vector marine_org_fld_name = { "TRUEPOLYC", "TRUEPROTC", "TRUELIPC"}; @@ -282,7 +282,6 @@ void MAMSrfOnlineEmiss::set_grids( // output morg_horizInterp_, morg_data_start_, morg_data_end_, morg_data_out_, morg_dataReader_); - printf("END SIZE:%i,%i\n", morg_data_end_.data.ncols, ncol_); } // set_grid ends @@ -452,6 +451,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { marineOrganicsFunc::marineOrganics_main(morg_timeState_, morg_data_start_, morg_data_end_, morg_data_out_); + // Marine organics emission data read from the file + const const_view_1d mpoly = ekat::subview(morg_data_out_.emiss_sectors, 0); + const const_view_1d mprot = ekat::subview(morg_data_out_.emiss_sectors, 1); + const const_view_1d mlip = ekat::subview(morg_data_out_.emiss_sectors, 2); + // FIXME: Remove the following vars as they are used only for validation const const_view_2d z_mid2 = get_field_in("z_mid").get_view(); // FIXME: Remove ^^^^^^ @@ -501,7 +505,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { mam4::aero_model_emissions::aero_model_emissions( sst(icol), ocnfrac(icol), u_wind(icol, surf_lev), v_wind(icol, surf_lev), z_mid(icol, surf_lev), dstflx_icol, - soil_erodibility(icol), fluxes_col); + soil_erodibility(icol), mpoly(icol), mprot(icol), mlip(icol), + // out + fluxes_col); }); // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes diff --git a/externals/mam4xx b/externals/mam4xx index 4f6259f61ae..7d6a52bec71 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 4f6259f61aeda3814df542e541a7a48de35e255e +Subproject commit 7d6a52bec71003de7da14433db1577042ff77218 From 5d8341f6402620be7b01347e7e91756ea17ce254 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 30 Oct 2024 21:18:57 -0700 Subject: [PATCH 338/366] EAMxx:Fixes units for marine organic emissions --- ...and_online_emissions_process_interface.cpp | 41 ++++++------------- ...and_online_emissions_process_interface.hpp | 2 - .../mam/readfiles/marine_organics_impl.hpp | 6 ++- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 4b3a7470d1f..a67ddf61604 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -50,9 +50,6 @@ void MAMSrfOnlineEmiss::set_grids( // For components of dust flux const FieldLayout vector4d = grid_->get_2d_vector_layout(4); - // FIXME: The following variables are used for validation ONLY!!! REMOVE - // THEM!!! - add_field("z_mid", scalar3d_m, kg / m2 / s, grid_name); // -------------------------------------------------------------------------- // These variables are "Required" or pure inputs for the process // -------------------------------------------------------------------------- @@ -79,8 +76,7 @@ void MAMSrfOnlineEmiss::set_grids( // Temperature[K] at midpoints add_field("T_mid", scalar3d_m, K, grid_name); - // Vertical pressure velocity [Pa/s] at midpoints (Require only for building - // DS) + // Vertical pressure velocity [Pa/s] at midpoints add_field("omega", scalar3d_m, Pa / s, grid_name); // Total pressure [Pa] at midpoints @@ -266,9 +262,9 @@ void MAMSrfOnlineEmiss::set_grids( // ------------------------------------------------------------- const std::string marine_organics_data_file = m_params.get("marine_organics_file"); - // /compyfs/inputdata/atm/cam/chem/trop_mam/marine_BGC/monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc - // Field to be read from file (order matters as they are read in same order) + // Field to be read from file (order matters as they are read in the same + // order) const std::vector marine_org_fld_name = { "TRUEPOLYC", "TRUEPROTC", "TRUELIPC"}; @@ -394,17 +390,11 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { //-------------------------------------------------------------------- // Update marine orgaincs from file //-------------------------------------------------------------------- - + // Time dependent data marineOrganicsFunc::update_marine_organics_data_from_file( morg_dataReader_, timestamp(), curr_month, *morg_horizInterp_, morg_data_end_); // output - //-------------------------------------------------------------------- - // Initialize online emissions from file - //-------------------------------------------------------------------- - online_emissions.online_emis_data.init(ncol_); - online_emissions.init_from_input_file(m_params); - //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- @@ -436,7 +426,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Online emissions from dust and sea salt //-------------------------------------------------------------------- - // --- Interpolate mariane organics data -- + // --- Interpolate marine organics data -- // Update TimeState, note the addition of dt morg_timeState_.t_now = ts.frac_of_year_in_days(); @@ -447,22 +437,15 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // output morg_timeState_, morg_data_start_, morg_data_end_); - // Call the main marineOrganics routine to get interpolated aerosol forcings. + // Call the main marine organics routine to get interpolated forcings. marineOrganicsFunc::marineOrganics_main(morg_timeState_, morg_data_start_, morg_data_end_, morg_data_out_); - // Marine organics emission data read from the file + // Marine organics emission data read from the file (order is important here) const const_view_1d mpoly = ekat::subview(morg_data_out_.emiss_sectors, 0); const const_view_1d mprot = ekat::subview(morg_data_out_.emiss_sectors, 1); const const_view_1d mlip = ekat::subview(morg_data_out_.emiss_sectors, 2); - // FIXME: Remove the following vars as they are used only for validation - const const_view_2d z_mid2 = get_field_in("z_mid").get_view(); - // FIXME: Remove ^^^^^^ - - // dust fluxes [kg/m^2/s]: Four flux values for each column - const const_view_2d dstflx = get_field_in("dstflx").get_view(); - // Ocean fraction [unitless] const const_view_1d ocnfrac = get_field_in("ocnfrac").get_view(); @@ -478,6 +461,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { const const_view_2d v_wind = get_field_in("horiz_winds").get_component(1).get_view(); + // Dust fluxes [kg/m^2/s]: Four flux values for each column + const const_view_2d dstflx = get_field_in("dstflx").get_view(); + // Constituent fluxes [kg/m^2/s] auto constituent_fluxes = this->constituent_fluxes_; @@ -490,11 +476,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // TODO: check that units are consistent with srf emissions! // copy current values to online-emissions-local version // TODO: potentially combine with below parfor(icol) loop? - const int surf_lev = nlev_ - 1; + const int surf_lev = nlev_ - 1; // surface level Kokkos::parallel_for( - // "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - "online_emis_fluxes", 1, KOKKOS_LAMBDA(int icol) { + "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { // input const const_view_1d dstflx_icol = ekat::subview(dstflx, icol); @@ -518,8 +503,6 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- // Interpolate srf emiss data read in from emissions files //-------------------------------------------------------------------- - // Gather time and state information for interpolation - // auto ts = timestamp() + dt; for(srf_emiss_ &ispec_srf : srf_emiss_species_) { // Update TimeState, note the addition of dt diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index fb162ee94e4..16cf6dcb74a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -172,8 +172,6 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // A vector for carrying emissions for all the species std::vector srf_emiss_species_; - onlineEmiss online_emissions; - // For reading marine organics file std::shared_ptr morg_horizInterp_; std::shared_ptr morg_dataReader_; diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp index 8f532d5342a..100b432cb74 100644 --- a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp @@ -54,7 +54,9 @@ marineOrganicsFunctions::create_horiz_remapper( const auto tgt_grid = remapper->get_tgt_grid(); const auto layout_2d = tgt_grid->get_2d_scalar_layout(); - const auto nondim = ekat::units::Units::nondimensional(); + using namespace ekat::units; + using namespace ekat::prefixes; + Units umolC(micro * mol, "umol C"); std::vector fields_vector; @@ -62,7 +64,7 @@ marineOrganicsFunctions::create_horiz_remapper( for(int icomp = 0; icomp < field_size; ++icomp) { auto comp_name = field_name[icomp]; // set and allocate fields - Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); + Field f(FieldIdentifier(comp_name, layout_2d, umolC, tgt_grid->name())); f.allocate_view(); fields_vector.push_back(f); remapper->register_field_from_tgt(f); From c681f075a5c5677417c4b7986a3722c1cee6caf9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 30 Oct 2024 21:58:08 -0700 Subject: [PATCH 339/366] EAMxx: Adds logic to zero out constituent fluxes, cleanup --- ...and_online_emissions_process_interface.cpp | 39 ++++--------------- ...and_online_emissions_process_interface.hpp | 21 +++++++--- externals/mam4xx | 2 +- 3 files changed, 25 insertions(+), 37 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index a67ddf61604..1d931435d06 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -413,12 +413,6 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // Zero-out output - // FIXME: Find out if we do it in the fortran code - // if we do, this should be a "Computed" field, - // ALSO this code should in the preprocessor struct now - Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); - // Gather time and state information for interpolation const auto ts = timestamp() + dt; @@ -468,25 +462,25 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { auto constituent_fluxes = this->constituent_fluxes_; // Soil edodibility [fraction] - auto soil_erodibility = this->soil_erodibility_; + const const_view_1d soil_erodibility = this->soil_erodibility_; // Vertical layer height at midpoints const const_view_2d z_mid = dry_atm_.z_mid; - // TODO: check that units are consistent with srf emissions! - // copy current values to online-emissions-local version // TODO: potentially combine with below parfor(icol) loop? const int surf_lev = nlev_ - 1; // surface level Kokkos::parallel_for( "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - // input + // Input const const_view_1d dstflx_icol = ekat::subview(dstflx, icol); - // output + // Output view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); - // comput online emissions + // Comput online emissions + // NOTE: mam4::aero_model_emissions calculates mass and number emission + // fluxes in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert mam4::aero_model_emissions::aero_model_emissions( sst(icol), ocnfrac(icol), u_wind(icol, surf_lev), v_wind(icol, surf_lev), z_mid(icol, surf_lev), dstflx_icol, @@ -494,11 +488,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // out fluxes_col); }); - - // NOTE: mam4::aero_model_emissions calculates mass and number emission fluxes - // in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert - // Kokkos::deep_copy(constituent_fluxes_, online_data.cfluxes); - // Kokkos::fence(); + Kokkos::fence(); //-------------------------------------------------------------------- // Interpolate srf emiss data read in from emissions files @@ -539,17 +529,4 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { } // for loop for species } // run_impl ends // ============================================================================= -} // namespace scream - -// NOTE: calling aero_model_emissions() this way hides the fact that -// some hard-coded data is initialized within mam4::aero_model_emissions -// some of these values definitely need to be read from here column-wise -// the structs.{values} holding the above are: -// SeasaltEmissionsData.{mpoly, mprot, mlip} -// DustEmissionsData.dust_dmt_vwr -// OnlineEmissionsData.{dust_flux_in, surface_temp, u_wind, -// v_wind, z_bottom, ocean_frac} -// NOTE: fortran mam4 gets dust_flux_in and ocean_frac from cam_in, -// which is a chunk-wise variable at this point -// (see: physpkg.F90:{1605 & 1327}) otherwise grabs the end/bottom -// col-value once inside aero_model_emissions() \ No newline at end of file +} // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 16cf6dcb74a..063efad22d4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -103,7 +103,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct Preprocess { Preprocess() = default; // on host: initializes preprocess functor with necessary state data - void initialize(const int ncol, const int nlev, + void initialize(const int &ncol, const int &nlev, const mam_coupling::WetAtmosphere &wet_atm, const mam_coupling::DryAtmosphere &dry_atm, const view_2d &constituent_fluxes) { @@ -116,14 +116,25 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index + const int icol = team.league_rank(); // column index - compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, icol); team.team_barrier(); // vertical heights has to be computed after computing dry mixing ratios // for atmosphere - compute_vertical_layer_heights(team, dry_atm_pre_, i); - compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + compute_vertical_layer_heights(team, dry_atm_pre_, icol); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, icol); + + view_1d flux_col = ekat::subview(constituent_fluxes_pre_, icol); + + // Zero out constituent fluxes only for gasses and aerosols + const int pcnst = mam4::aero_model::pcnst; + const int gas_start_ind = mam4::utils::gasses_start_ind(); + + // FIXME: Is there a better way to zero out a select indices? + for(int ispc = gas_start_ind; ispc < pcnst; ++ispc) { + flux_col(ispc) = 0; + } } // Preprocess operator() // local variables for preprocess struct diff --git a/externals/mam4xx b/externals/mam4xx index 7d6a52bec71..419e04b4943 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 7d6a52bec71003de7da14433db1577042ff77218 +Subproject commit 419e04b4943ac4a69d867f6087ca43652a0689a2 From a150d9d53ad5c389ffe4db178d99bd6265e67278 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 30 Oct 2024 22:18:26 -0700 Subject: [PATCH 340/366] EAMxx: Reverts input yaml and CMake, changes cons fluxes to updated --- ...and_online_emissions_process_interface.cpp | 5 ++- .../mam/emissions/CMakeLists.txt | 3 +- .../single-process/mam/emissions/input.yaml | 36 ++----------------- 3 files changed, 6 insertions(+), 38 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 1d931435d06..f849595ff34 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,4 +1,3 @@ - #include // For reading soil erodibility file @@ -124,8 +123,8 @@ void MAMSrfOnlineEmiss::set_grids( // Constituent fluxes of species in [kg/m2/s] // FIXME: confirm if it is Updated or Computed - add_field("constituent_fluxes", vector2d_pcnst, kg / m2 / s, - grid_name); + add_field("constituent_fluxes", vector2d_pcnst, kg / m2 / s, + grid_name); // Surface emissions remapping file auto srf_map_file = m_params.get("srf_remap_file", ""); diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index 4b7350a1bb9..65f377b4db1 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -13,8 +13,7 @@ CreateADUnitTest(${TEST_BASE_NAME} # Set AD configurable options set (ATM_TIME_STEP 1800) -#SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h -SetVarDependingOnTestSize(NUM_STEPS 1 1 1) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h set (RUN_T0 2021-10-12-45000) ## Copy (and configure) yaml files needed by tests diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index ce103765b70..64a07b84405 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -25,34 +25,6 @@ atmosphere_processes: soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc marine_organics_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc - # FIXME: just set everything to a constant scalar for testing - online_emis_IC_ncl_a1: 0.0 - online_emis_IC_ncl_a2: 0.0 - online_emis_IC_ncl_a3: 0.0 - online_emis_IC_mom_a1: 0.16838301005552275E-013 - online_emis_IC_mom_a2: 0.26554873160224250E-016 - online_emis_IC_mom_a4: 0.0 - online_emis_IC_num_a1: 0.0 - online_emis_IC_num_a2: 0.0 - online_emis_IC_num_a3: 0.0 - online_emis_IC_num_a4: 0.0 - online_emis_IC_dst_a1: 0.0 - online_emis_IC_dst_a3: 0.18720550166902007E+003 - -# NOTE: these are the single-call results from the mam validation test -# (20) ncl_a1: 2.659900637e-13 -# (25) ncl_a2: 0 -# (29) ncl_a3: 2.391492482e-11 -# (21) mom_a1: 1.683830101e-14 -# (26) mom_a2: 2.655487316e-17 -# (38) mom_a4: 0 -# (22) num_a1: 2.594942753e-15 -# (27) num_a2: 4.092359179e-18 -# (35) num_a3: 0 -# (39) num_a4: 0 -# (19) dst_a1: 0 -# (28) dst_a3: 403.2611563 - grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -65,12 +37,10 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - #Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} - Filename: /qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/4src/screami_unit_tests_mam4xx_ne2np4L72_LAT_m5p57727434969464_LON_83p9312243922771_ONLINE_EMISS_1st.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 - #These should come from the input file - + #we should get the following variables from other processes pbl_height : 1.0 dstflx : [ 0.00000000000000000E+000, 0.00000000000000000E+000, 0.00000000000000000E+000, 0.00000000000000000E+000] @@ -78,7 +48,7 @@ initial_conditions: sst: 0.30178553874977507E+003 cldfrac_tot: 0.138584624960092 horiz_winds: [-0.24988988196194634E+000, -0.23959782871450760E+000] - + constituent_fluxes: 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From f0a33da27edd95cfde8d670cfe73521d93f041ca Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 30 Oct 2024 22:33:32 -0700 Subject: [PATCH 341/366] EAMxx: Some fixes after rebase --- .../eamxx_mam_microphysics_process_interface.cpp | 2 +- .../eamxx/src/physics/mam/srf_emission_impl.hpp | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 41713b97735..5fb4dccb2c8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -76,7 +76,7 @@ void MAMMicrophysics::set_grids( constexpr auto n_unit = 1 / kg; // units of number mixing ratios of tracers ======= config_.amicphys.nucleation = {}; - // config_.amicphys.nucleation.dens_so4a_host = 1770.0; + //config_.amicphys.nucleation.dens_so4a_host = 1770.0; // config_.amicphys.nucleation.mw_so4a_host = 115.0; // config_.amicphys.nucleation.newnuc_method_user_choice = 2; // config_.amicphys.nucleation.pbl_nuc_wang2008_user_choice = 1; diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b090e3746cb..4e7f7c14424 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -182,10 +182,7 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int time_index, // zero-based AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input) { using namespace ShortFieldTagsNames; - // NOTE: these are currently unused - // using ESU = ekat::ExeSpaceUtils; - // using Member = typename KokkosTypes::MemberType; - + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); // 1. Read from file @@ -205,14 +202,6 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - // NOTE: these are currently unused - // const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) - // .get_header() - // .get_identifier() - // .get_layout(); - - const int ncols = layout.dim(COL); - // Read fields from the file for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = From 80a57f053ecf36f4737d4425c08f512de573f2e4 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 30 Oct 2024 23:02:18 -0700 Subject: [PATCH 342/366] EAMxx:Adds loop to update file read for marine organics, cleanup --- .../cime_config/namelist_defaults_scream.xml | 2 - ...and_online_emissions_process_interface.cpp | 5 +- ...and_online_emissions_process_interface.hpp | 4 +- .../eamxx/src/physics/mam/online_emission.hpp | 88 ------------------- .../mam/readfiles/marine_organics_impl.hpp | 17 ++-- .../src/physics/mam/srf_emission_impl.hpp | 3 - 6 files changed, 16 insertions(+), 103 deletions(-) delete mode 100644 components/eamxx/src/physics/mam/online_emission.hpp diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 8b3a1131d2a..b57a5746253 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -612,7 +612,6 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_CA_ne32_x32_v1_pg2_16xdel2.nc - ${CASE}.scream 0.0 0.0 @@ -698,7 +697,6 @@ be lost if SCREAM_HACK_XML is not enabled. - ./${CASE}.scream default ${REST_N} diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f849595ff34..190dc1da27d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -247,7 +247,7 @@ void MAMSrfOnlineEmiss::set_grids( // Field to be read from file const std::string soil_erod_fld_name = "mbl_bsn_fct_geo"; - // Dimensions of the filed + // Dimensions of the field const std::string soil_erod_dname = "ncol"; // initialize the file read @@ -267,7 +267,7 @@ void MAMSrfOnlineEmiss::set_grids( const std::vector marine_org_fld_name = { "TRUEPOLYC", "TRUEPROTC", "TRUELIPC"}; - // Dimensions of the filed + // Dimensions of the field const std::string marine_org_dname = "ncol"; // initialize the file read @@ -526,6 +526,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); }); } // for loop for species + Kokkos::fence(); } // run_impl ends // ============================================================================= } // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 063efad22d4..b1cba682bb6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -6,7 +6,6 @@ // For MAM4 aerosol configuration #include -#include #include // For reading marine organics file @@ -62,9 +61,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { const_view_1d soil_erodibility_; public: + // For reading surface emissions and marine organics file using srfEmissFunc = mam_coupling::srfEmissFunctions; - using onlineEmiss = mam_coupling::onlineEmissions; - // For reading marine organics file using marineOrganicsFunc = marine_organics::marineOrganicsFunctions; diff --git a/components/eamxx/src/physics/mam/online_emission.hpp b/components/eamxx/src/physics/mam/online_emission.hpp deleted file mode 100644 index c8318dcc05c..00000000000 --- a/components/eamxx/src/physics/mam/online_emission.hpp +++ /dev/null @@ -1,88 +0,0 @@ -#ifndef ONLINE_EMISSION_HPP -#define ONLINE_EMISSION_HPP - -#include "share/util/scream_timing.hpp" - -namespace scream::mam_coupling { -template struct onlineEmissions { - using Device = DeviceType; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - static constexpr int pcnst = mam4::aero_model::pcnst; - - struct onlineEmissData { - // Basic spatial dimensions of the data - int ncols; - view_2d flux_data; - // local copy of main fluxes array - view_2d cfluxes; - // FIXME: read this from elsewhere? input? - const std::vector spec_names = { - "ncl_a1", "ncl_a2", "ncl_a3", "mom_a1", "mom_a2", "mom_a4", - "num_a1", "num_a2", "num_a3", "num_a4", "dst_a1", "dst_a3"}; - // FIXME: change this when/if the above is dynamically-determined - int nspec = spec_names.size(); - const std::string root_IC_str = "online_emis_IC_"; - - onlineEmissData() = default; - onlineEmissData(const int ncol_, const int nspec_) - : ncols(ncol_), nspec(nspec_) { - init(ncols, nspec, true); - } - - onlineEmissData(const int ncol_) : ncols(ncol_) { - init(ncols, nspec, true); - } - - // overloads of init() in case npsec is not hard-coded, or if you want to - // control allocation via bool flag - void init(const int ncol_, const int nspec_, const bool allocate_) { - ncols = ncol_; - nspec = nspec_; - if (allocate_) { - flux_data = view_2d("onlineEmissData", nspec, ncols); - cfluxes = view_2d("onlineEmisLocalCflux", ncols, pcnst); - } - } // onlineEmissData init - void init(const int ncol_, const bool allocate_) { - ncols = ncol_; - if (allocate_) { - flux_data = view_2d("onlineEmissData", nspec, ncols); - cfluxes = view_2d("onlineEmisLocalCflux", ncols, pcnst); - } - } // onlineEmissData init - void init(const int ncol_) { - ncols = ncol_; - flux_data = view_2d("onlineEmissData", nspec, ncols); - cfluxes = view_2d("onlineEmisLocalCflux", ncols, pcnst); - } // onlineEmissData init - }; // onlineEmissData - - onlineEmissData online_emis_data; - - // --------------------------------------------------------------------------- - // Online emissions routines - // --------------------------------------------------------------------------- - void init_from_input_file(const ekat::ParameterList ¶ms) { - const int nspec = online_emis_data.nspec; - const int ncols = online_emis_data.ncols; - using ExeSpace = typename KT::ExeSpace; - using ESU = ekat::ExeSpaceUtils; - const auto policy = ESU::get_default_team_policy(ncols, nspec); - // Read from input file - // FIXME: currently reading a single placeholder scalar--should be - // ncols-sized array when we know what the input data looks like - for (int ispec = 0; ispec < nspec; ++ispec) { - Real init_cond_val = - params.get(online_emis_data.root_IC_str + online_emis_data.spec_names[ispec]); - // TODO: is this overkill?--i.e., would a mirror/deep_copy make more sense? - Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const MemberType &team) { - const int jcol = team.league_rank(); // column index - online_emis_data.flux_data(ispec, jcol) = init_cond_val; - }); - } - } // end init_from_input_file() -}; // struct onlineEmissions -} // namespace scream::mam_coupling -#endif // ONLINE_EMISSION_HPP diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp index 100b432cb74..6eda3a47e72 100644 --- a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp @@ -85,8 +85,8 @@ marineOrganicsFunctions::create_data_reader( const std::shared_ptr &horiz_remapper, const std::string &data_file) { std::vector io_fields; - for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - io_fields.push_back(horiz_remapper->get_src_field(i)); + for(int ifld = 0; ifld < horiz_remapper->get_num_fields(); ++ifld) { + io_fields.push_back(horiz_remapper->get_src_field(ifld)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(data_file, io_grid, io_fields, true); @@ -126,9 +126,16 @@ void marineOrganicsFunctions::update_marine_organics_data_from_file( "field"); // Recall, the fields are registered in the order: // Read the field from the file -#if 0 - input = horiz_interp.get_tgt_field(0).get_view(); -#endif + + for(int ifld = 0; ifld < horiz_interp.get_num_fields(); ++ifld) { + auto sector = horiz_interp.get_tgt_field(ifld).get_view(); + const auto emiss = Kokkos::subview(marineOrganics_input.data.emiss_sectors, + ifld, Kokkos::ALL()); + Kokkos::deep_copy(emiss, sector); + } + + Kokkos::fence(); + stop_timer( "EAMxx::marineOrganics::update_marine_organics_data_from_file::get_" "field"); diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 4e7f7c14424..fa037dc281d 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -102,9 +102,6 @@ void srfEmissFunctions::perform_time_interpolation( // NOTE: we *assume* data_beg and data_end have the *same* hybrid v coords. // IF this ever ceases to be the case, you can interp those too. - using ExeSpace = typename KT::ExeSpace; - using ESU = ekat::ExeSpaceUtils; - // Gather time stamp info auto &t_now = time_state.t_now; auto &t_beg = time_state.t_beg_month; From 15f44fd3101cd3a67a634af34e498fc1736362e7 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Thu, 31 Oct 2024 16:43:23 -0600 Subject: [PATCH 343/366] fixup! EAMxx: Some fixes after rebase small change to kickoff AT --- components/eamxx/tests/single-process/mam/emissions/input.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 64a07b84405..a4564cb3949 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -41,7 +41,7 @@ initial_conditions: topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 - #we should get the following variables from other processes + # we should get the following variables from other processes pbl_height : 1.0 dstflx : [ 0.00000000000000000E+000, 0.00000000000000000E+000, 0.00000000000000000E+000, 0.00000000000000000E+000] ocnfrac: 0.10000000000000000E+001 From bdd475f60e4fbd7703159f2e1217f43391f6623b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 1 Nov 2024 06:59:01 -0700 Subject: [PATCH 344/366] EAMxx:Fixes file path and specify vector components for dust flux --- components/eamxx/cime_config/namelist_defaults_scream.xml | 7 ++++++- components/eamxx/src/mct_coupling/scream_cpl_indices.F90 | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index b57a5746253..9cc917a8c88 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -376,12 +376,17 @@ be lost if SCREAM_HACK_XML is not enabled. ======= - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/dst_ne30pg2_c20241028.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/dst_ne30pg2_c20241028.nc +<<<<<<< HEAD ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/dst_ne4pg2_c20241028.nc >>>>>>> EAMxx:Adds soil erodibility file read and sent to online emission read +======= + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/dst_ne4pg2_c20241028.nc + +>>>>>>> EAMxx:Fixes file path and specify vector components for dust flux ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne30pg2_c20241030.nc diff --git a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 index 15df178d29e..9462750297f 100644 --- a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 +++ b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 @@ -125,12 +125,16 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_cpl_indices(22) = mct_avect_indexra(x2a,'Fall_flxdst2') import_cpl_indices(23) = mct_avect_indexra(x2a,'Fall_flxdst3') import_cpl_indices(24) = mct_avect_indexra(x2a,'Fall_flxdst4') - - ! Vector components + !(Faxx_taux and Faxx_tauy) import_vector_components(11) = 0 import_vector_components(12) = 1 + !(dust fluxes) + import_vector_components(21) = 0 + import_vector_components(22) = 1 + import_vector_components(23) = 2 + import_vector_components(24) = 3 ! Constant multiples import_constant_multiple(10) = -1 From 4e120c747c038559b90f9424e895f6795c7c4cfd Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 4 Nov 2024 11:06:50 -0800 Subject: [PATCH 345/366] EAMxx: MAM4xx submodule pointing to main that include GPU fixes --- ...mxx_mam_microphysics_process_interface.cpp | 102 ++++++++++++++++++ externals/mam4xx | 2 +- 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 5fb4dccb2c8..875d63432f8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -611,6 +611,7 @@ void MAMMicrophysics::run_impl(const double dt) { // allocation perspective auto o3_col_dens = buffer_.scratch[8]; +<<<<<<< HEAD /* Gather time and state information for interpolation */ const auto ts = timestamp() + dt; @@ -628,6 +629,107 @@ void MAMMicrophysics::run_impl(const double dt) { dry_atm_.p_mid, dry_atm_.z_iface, // in cnst_offline_); // out Kokkos::fence(); +======= + const_view_1d &col_latitudes = col_latitudes_; + mam_coupling::DryAtmosphere &dry_atm = dry_atm_; + mam_coupling::AerosolState &dry_aero = dry_aero_; + mam4::mo_photo::PhotoTableData &photo_table = photo_table_; + const int nlev = nlev_; + const Config &config = config_; + // FIXME: read relevant linoz climatology data from file(s) based on time + + // FIXME: read relevant chlorine loading data from file based on time + + // loop over atmosphere columns and compute aerosol microphyscs + auto some_step = step_; + + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { + const int icol = team.league_rank(); // column index + + Real col_lat = col_latitudes(icol); // column latitude (degrees?) + + // fetch column-specific atmosphere state data + auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); + auto z_iface = ekat::subview(dry_atm.z_iface, icol); + Real phis = dry_atm.phis(icol); + + // set surface state data + haero::Surface sfc{}; + + // fetch column-specific subviews into aerosol prognostics + mam4::Prognostics progs = mam_coupling::interstitial_aerosols_for_column(dry_aero, icol); + + // set up diagnostics + mam4::Diagnostics diags(nlev); + + // calculate o3 column densities (first component of col_dens in Fortran code) + auto o3_col_dens_i = ekat::subview(o3_col_dens, icol); + impl::compute_o3_column_density(team, atm, progs, o3_col_dens_i); + + // set up photolysis work arrays for this column. + mam4::mo_photo::PhotoTableWorkArrays photo_work_arrays; + // FIXME: set views here + + // ... look up photolysis rates from our table + // NOTE: the table interpolation operates on an entire column of data, so we + // NOTE: must do it before dispatching to individual vertical levels + Real zenith_angle = 0.0; // FIXME: need to get this from EAMxx [radians] + Real surf_albedo = 0.0; // FIXME: surface albedo + Real esfact = 0.0; // FIXME: earth-sun distance factor + mam4::ColumnView lwc; // FIXME: liquid water cloud content: where do we get this? + //mam4::mo_photo::table_photo(photo_rates, atm.pressure, atm.hydrostatic_dp, + // atm.temperature, o3_col_dens_i, zenith_angle, surf_albedo, lwc, + // atm.cloud_fraction, esfact, photo_table, photo_work_arrays); + + // compute external forcings at time t(n+1) [molecules/cm^3/s] + constexpr int extcnt = mam4::gas_chemistry::extcnt; + view_2d extfrc; // FIXME: where to allocate? (nlev, extcnt) + mam4::mo_setext::Forcing forcings[extcnt]; // FIXME: forcings seem to require file data + mam4::mo_setext::extfrc_set(forcings, extfrc); + + // compute aerosol microphysics on each vertical level within this column + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [&](const int k) { + + constexpr int num_modes = mam4::AeroConfig::num_modes(); + constexpr int gas_pcnst = mam_coupling::gas_pcnst(); + constexpr int nqtendbb = mam_coupling::nqtendbb(); + + // extract atm state variables (input) + Real temp = atm.temperature(k); + Real pmid = atm.pressure(k); + Real pdel = atm.hydrostatic_dp(k); + Real zm = atm.height(k); + Real zi = z_iface(k); + Real pblh = atm.planetary_boundary_layer_height; + Real qv = atm.vapor_mixing_ratio(k); + Real cldfrac = atm.cloud_fraction(k); + + // extract aerosol state variables into "working arrays" (mass mixing ratios) + // (in EAM, this is done in the gas_phase_chemdr subroutine defined within + // mozart/mo_gas_phase_chemdr.F90) + Real q[gas_pcnst] = {}; + Real qqcw[gas_pcnst] = {}; + mam_coupling::transfer_prognostics_to_work_arrays(progs, k, q, qqcw); + + // convert mass mixing ratios to volume mixing ratios (VMR), equivalent + // to tracer mixing ratios (TMR)) + Real vmr[gas_pcnst], vmrcw[gas_pcnst]; + mam_coupling::convert_work_arrays_to_vmr(q, qqcw, vmr, vmrcw); + + // aerosol/gas species tendencies (output) + Real vmr_tendbb[gas_pcnst][nqtendbb] = {}; + Real vmrcw_tendbb[gas_pcnst][nqtendbb] = {}; + + // create work array copies to retain "pre-chemistry" values + Real vmr_pregaschem[gas_pcnst] = {}; + Real vmr_precldchem[gas_pcnst] = {}; + Real vmrcw_precldchem[gas_pcnst] = {}; + for (int i = 0; i < gas_pcnst; ++i) { + vmr_pregaschem[i] = vmr[i]; + vmr_precldchem[i] = vmr[i]; + vmrcw_precldchem[i] = vmrcw[i]; + } +>>>>>>> EAMxx: MAM4xx submodule pointing to main that include GPU fixes scream::mam_coupling::advance_tracer_data( LinozDataReader_, // in diff --git a/externals/mam4xx b/externals/mam4xx index 419e04b4943..59990d402f4 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 419e04b4943ac4a69d867f6087ca43652a0689a2 +Subproject commit 59990d402f4115ce3a36a8c00eb4d059e703a583 From 7b8f7fdd2871018dabeeae8a53dd76f4333905f9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 4 Nov 2024 13:26:15 -0800 Subject: [PATCH 346/366] EAMxx: Some codes re-arranged after rebase --- .../cime_config/namelist_defaults_scream.xml | 9 -- ...mxx_mam_microphysics_process_interface.cpp | 113 ------------------ 2 files changed, 122 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 9cc917a8c88..fd62cdc01e3 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -371,22 +371,13 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc -<<<<<<< HEAD -======= - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/dst_ne30pg2_c20241028.nc -<<<<<<< HEAD ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/dst_ne4pg2_c20241028.nc ->>>>>>> EAMxx:Adds soil erodibility file read and sent to online emission read -======= - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/dst_ne4pg2_c20241028.nc - ->>>>>>> EAMxx:Fixes file path and specify vector components for dust flux ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne30pg2_c20241030.nc diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 875d63432f8..57eaf927548 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -70,20 +70,9 @@ void MAMMicrophysics::set_grids( const FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); const FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); -<<<<<<< HEAD using namespace ekat::units; constexpr auto q_unit = kg / kg; // units of mass mixing ratios of tracers constexpr auto n_unit = 1 / kg; // units of number mixing ratios of tracers -======= - config_.amicphys.nucleation = {}; - //config_.amicphys.nucleation.dens_so4a_host = 1770.0; - // config_.amicphys.nucleation.mw_so4a_host = 115.0; - // config_.amicphys.nucleation.newnuc_method_user_choice = 2; - // config_.amicphys.nucleation.pbl_nuc_wang2008_user_choice = 1; - // config_.amicphys.nucleation.adjust_factor_pbl_ratenucl = 1.0; - // config_.amicphys.nucleation.accom_coef_h2so4 = 1.0; - config_.amicphys.nucleation.newnuc_adjust_factor_dnaitdt = 1.0; ->>>>>>> comment out error-inducing code in mam4_amicphys.cpp and small reorg change in online_emission.hpp // -------------------------------------------------------------------------- // These variables are "Required" or pure inputs for the process @@ -611,7 +600,6 @@ void MAMMicrophysics::run_impl(const double dt) { // allocation perspective auto o3_col_dens = buffer_.scratch[8]; -<<<<<<< HEAD /* Gather time and state information for interpolation */ const auto ts = timestamp() + dt; @@ -629,107 +617,6 @@ void MAMMicrophysics::run_impl(const double dt) { dry_atm_.p_mid, dry_atm_.z_iface, // in cnst_offline_); // out Kokkos::fence(); -======= - const_view_1d &col_latitudes = col_latitudes_; - mam_coupling::DryAtmosphere &dry_atm = dry_atm_; - mam_coupling::AerosolState &dry_aero = dry_aero_; - mam4::mo_photo::PhotoTableData &photo_table = photo_table_; - const int nlev = nlev_; - const Config &config = config_; - // FIXME: read relevant linoz climatology data from file(s) based on time - - // FIXME: read relevant chlorine loading data from file based on time - - // loop over atmosphere columns and compute aerosol microphyscs - auto some_step = step_; - - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { - const int icol = team.league_rank(); // column index - - Real col_lat = col_latitudes(icol); // column latitude (degrees?) - - // fetch column-specific atmosphere state data - auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); - auto z_iface = ekat::subview(dry_atm.z_iface, icol); - Real phis = dry_atm.phis(icol); - - // set surface state data - haero::Surface sfc{}; - - // fetch column-specific subviews into aerosol prognostics - mam4::Prognostics progs = mam_coupling::interstitial_aerosols_for_column(dry_aero, icol); - - // set up diagnostics - mam4::Diagnostics diags(nlev); - - // calculate o3 column densities (first component of col_dens in Fortran code) - auto o3_col_dens_i = ekat::subview(o3_col_dens, icol); - impl::compute_o3_column_density(team, atm, progs, o3_col_dens_i); - - // set up photolysis work arrays for this column. - mam4::mo_photo::PhotoTableWorkArrays photo_work_arrays; - // FIXME: set views here - - // ... look up photolysis rates from our table - // NOTE: the table interpolation operates on an entire column of data, so we - // NOTE: must do it before dispatching to individual vertical levels - Real zenith_angle = 0.0; // FIXME: need to get this from EAMxx [radians] - Real surf_albedo = 0.0; // FIXME: surface albedo - Real esfact = 0.0; // FIXME: earth-sun distance factor - mam4::ColumnView lwc; // FIXME: liquid water cloud content: where do we get this? - //mam4::mo_photo::table_photo(photo_rates, atm.pressure, atm.hydrostatic_dp, - // atm.temperature, o3_col_dens_i, zenith_angle, surf_albedo, lwc, - // atm.cloud_fraction, esfact, photo_table, photo_work_arrays); - - // compute external forcings at time t(n+1) [molecules/cm^3/s] - constexpr int extcnt = mam4::gas_chemistry::extcnt; - view_2d extfrc; // FIXME: where to allocate? (nlev, extcnt) - mam4::mo_setext::Forcing forcings[extcnt]; // FIXME: forcings seem to require file data - mam4::mo_setext::extfrc_set(forcings, extfrc); - - // compute aerosol microphysics on each vertical level within this column - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [&](const int k) { - - constexpr int num_modes = mam4::AeroConfig::num_modes(); - constexpr int gas_pcnst = mam_coupling::gas_pcnst(); - constexpr int nqtendbb = mam_coupling::nqtendbb(); - - // extract atm state variables (input) - Real temp = atm.temperature(k); - Real pmid = atm.pressure(k); - Real pdel = atm.hydrostatic_dp(k); - Real zm = atm.height(k); - Real zi = z_iface(k); - Real pblh = atm.planetary_boundary_layer_height; - Real qv = atm.vapor_mixing_ratio(k); - Real cldfrac = atm.cloud_fraction(k); - - // extract aerosol state variables into "working arrays" (mass mixing ratios) - // (in EAM, this is done in the gas_phase_chemdr subroutine defined within - // mozart/mo_gas_phase_chemdr.F90) - Real q[gas_pcnst] = {}; - Real qqcw[gas_pcnst] = {}; - mam_coupling::transfer_prognostics_to_work_arrays(progs, k, q, qqcw); - - // convert mass mixing ratios to volume mixing ratios (VMR), equivalent - // to tracer mixing ratios (TMR)) - Real vmr[gas_pcnst], vmrcw[gas_pcnst]; - mam_coupling::convert_work_arrays_to_vmr(q, qqcw, vmr, vmrcw); - - // aerosol/gas species tendencies (output) - Real vmr_tendbb[gas_pcnst][nqtendbb] = {}; - Real vmrcw_tendbb[gas_pcnst][nqtendbb] = {}; - - // create work array copies to retain "pre-chemistry" values - Real vmr_pregaschem[gas_pcnst] = {}; - Real vmr_precldchem[gas_pcnst] = {}; - Real vmrcw_precldchem[gas_pcnst] = {}; - for (int i = 0; i < gas_pcnst; ++i) { - vmr_pregaschem[i] = vmr[i]; - vmr_precldchem[i] = vmr[i]; - vmrcw_precldchem[i] = vmrcw[i]; - } ->>>>>>> EAMxx: MAM4xx submodule pointing to main that include GPU fixes scream::mam_coupling::advance_tracer_data( LinozDataReader_, // in From 4195b419bee625fb962646433b58ca3c4747f252 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 4 Nov 2024 13:29:00 -0800 Subject: [PATCH 347/366] EAMxx:Removes accidently added file --- .../src/physics/mam/impl/mam4_amicphys.cpp | 1819 ----------------- 1 file changed, 1819 deletions(-) delete mode 100644 components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp diff --git a/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp b/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp deleted file mode 100644 index e241b5ec428..00000000000 --- a/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp +++ /dev/null @@ -1,1819 +0,0 @@ -#include -#include -#include -#include -#include -#include - -namespace scream::impl { - -#define MAX_FILENAME_LEN 256 - -using namespace mam4; - -// number of constituents in gas chemistry "work arrays" -KOKKOS_INLINE_FUNCTION -constexpr int gas_pcnst() { - constexpr int gas_pcnst_ = mam4::gas_chemistry::gas_pcnst; - return gas_pcnst_; -} - -// number of aerosol/gas species tendencies -KOKKOS_INLINE_FUNCTION -constexpr int nqtendbb() { return 4; } - -// MAM4 aerosol microphysics configuration data -struct AmicPhysConfig { - // these switches activate various aerosol microphysics processes - bool do_cond; // condensation (a.k.a gas-aerosol exchange) - bool do_rename; // mode "renaming" - bool do_newnuc; // gas -> aerosol nucleation - bool do_coag; // aerosol coagulation - - // configurations for specific aerosol microphysics - mam4::GasAerExchProcess::ProcessConfig condensation; - mam4::NucleationProcess::ProcessConfig nucleation; - - // controls treatment of h2so4 condensation in mam_gasaerexch_1subarea - // 1 = sequential calc. of gas-chem prod then condensation loss - // 2 = simultaneous calc. of gas-chem prod and condensation loss - int gaexch_h2so4_uptake_optaa; - - // controls how nucleation interprets h2so4 concentrations - int newnuc_h2so4_conc_optaa; -}; - -namespace { - -KOKKOS_INLINE_FUNCTION constexpr int nqtendaa() { return 5; } -KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendaa() { return 1; } -KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendbb() { return 1; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond() { return 0; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_rnam() { return 1; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_nnuc() { return 2; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_coag() { return 3; } -KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond_only() { return 4; } -KOKKOS_INLINE_FUNCTION constexpr int iqqcwtend_rnam() { return 0; } -KOKKOS_INLINE_FUNCTION constexpr int maxsubarea() { return 2; } - -// conversion factors -KOKKOS_INLINE_FUNCTION Real fcvt_gas(int gas_id) { - static const Real fcvt_gas_[AeroConfig::num_gas_ids()] = {1, 1, 1}; - return fcvt_gas_[gas_id]; -} -KOKKOS_INLINE_FUNCTION Real fcvt_aer(int aero_id) { - static const Real fcvt_aer_[AeroConfig::num_aerosol_ids()] = {1, 1, 1, 1, 1, 1, 1}; - return fcvt_aer_[aero_id]; -} - -// leave number mix-ratios unchanged (#/kmol-air) -KOKKOS_INLINE_FUNCTION Real fcvt_num() { return 1.0; } -// factor for converting aerosol water mix-ratios from (kg/kg) to (mol/mol) -KOKKOS_INLINE_FUNCTION Real fcvt_wtr() { return 1.0; } - -KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_nul() { return 0; } -KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_gas() { return 1; } -KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_aer() { return 2; } -KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_num() { return 3; } -KOKKOS_INLINE_FUNCTION int lmapcc_all(int index) { - static const int lmapcc_all_[gas_pcnst()] = { - lmapcc_val_nul(), lmapcc_val_gas(), lmapcc_val_nul(), lmapcc_val_nul(), - lmapcc_val_gas(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), - lmapcc_val_aer(), lmapcc_val_num()}; - return lmapcc_all_[index]; -} - -// Where lmapcc_val_num are defined in lmapcc_all -KOKKOS_INLINE_FUNCTION int numptr_amode(int mode) { - static const int numptr_amode_[AeroConfig::num_modes()] = {12, 17, 25, 29}; - return numptr_amode_[mode]; -} - -// Where lmapcc_val_gas are defined in lmapcc_all -KOKKOS_INLINE_FUNCTION int lmap_gas(int mode) { - static const int lmap_gas_[AeroConfig::num_modes()] = {4, 1}; - return lmap_gas_[mode]; -} -// Where lmapcc_val_aer are defined in lmapcc_all -KOKKOS_INLINE_FUNCTION int lmassptr_amode(int aero_id, int mode) { - static const int lmassptr_amode_[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] = { - {5, 13, 18, 26}, {6, 14, 19, 27}, {7, 15, 20, 28}, {8, 16, 21, -6}, - {9, -6, 22, -6}, {10, -6, 23, -6}, {11, -6, 24, -6}}; - return lmassptr_amode_[aero_id][mode]; -} - -KOKKOS_INLINE_FUNCTION -void subarea_partition_factors( - const Real - q_int_cell_avg, // in grid cell mean interstitial aerosol mixing ratio - const Real - q_cbn_cell_avg, // in grid cell mean cloud-borne aerosol mixing ratio - const Real fcldy, // in cloudy fraction of the grid cell - const Real fclea, // in clear fraction of the grid cell - Real &part_fac_q_int_clea, // out - Real &part_fac_q_int_cldy) // out -{ - // Calculate mixing ratios of each subarea - - // cloud-borne, cloudy subarea - const Real tmp_q_cbn_cldy = q_cbn_cell_avg / fcldy; - // interstitial, cloudy subarea - const Real tmp_q_int_cldy = - haero::max(0.0, ((q_int_cell_avg + q_cbn_cell_avg) - tmp_q_cbn_cldy)); - // interstitial, clear subarea - const Real tmp_q_int_clea = (q_int_cell_avg - fcldy * tmp_q_int_cldy) / fclea; - - // Calculate the corresponding paritioning factors for interstitial aerosols - // using the above-derived subarea mixing ratios plus the constraint that - // the cloud fraction weighted average of subarea mean need to match grid box - // mean. - - // *** question *** - // use same part_fac_q_int_clea/cldy for everything ? - // use one for number and one for all masses (based on total mass) ? - // use separate ones for everything ? - // maybe one for number and one for all masses is best, - // because number and mass have different activation fractions - // *** question *** - - Real tmp_aa = haero::max(1.e-35, tmp_q_int_clea * fclea) / - haero::max(1.e-35, q_int_cell_avg); - tmp_aa = haero::max(0.0, haero::min(1.0, tmp_aa)); - - part_fac_q_int_clea = tmp_aa / fclea; - part_fac_q_int_cldy = (1.0 - tmp_aa) / fcldy; -} - -KOKKOS_INLINE_FUNCTION -void construct_subareas_1gridcell( - const Real cld, // in - const Real relhumgcm, // in - const Real q_pregaschem[gas_pcnst()], // in q TMRs before - // gas-phase chemistry - const Real q_precldchem[gas_pcnst()], // in q TMRs before - // cloud chemistry - const Real qqcw_precldchem[gas_pcnst()], // in qqcw TMRs before - // cloud chemistry - const Real q[gas_pcnst()], // in current tracer mixing ratios (TMRs) - // *** MUST BE #/kmol-air for number - // *** MUST BE mol/mol-air for mass - const Real qqcw[gas_pcnst()], // in like q but for - // cloud-borner tracers - int &nsubarea, // out - int &ncldy_subarea, // out - int &jclea, // out - int &jcldy, // out - bool iscldy_subarea[maxsubarea()], // out - Real afracsub[maxsubarea()], // out - Real relhumsub[maxsubarea()], // out - Real qsub1[gas_pcnst()][maxsubarea()], // out interstitial - Real qsub2[gas_pcnst()][maxsubarea()], // out interstitial - Real qsub3[gas_pcnst()][maxsubarea()], // out interstitial - Real qqcwsub1[gas_pcnst()][maxsubarea()], // out cloud-borne - Real qqcwsub2[gas_pcnst()][maxsubarea()], // out cloud-borne - Real qqcwsub3[gas_pcnst()][maxsubarea()], // outcloud-borne - Real qaerwatsub3[AeroConfig::num_modes()] - [maxsubarea()], // out aerosol water mixing ratios (mol/mol) - Real qaerwat[AeroConfig::num_modes()] // in aerosol water mixing ratio - // (kg/kg, NOT mol/mol) -) { - static constexpr int num_modes = AeroConfig::num_modes(); - // cloud chemistry is only on when cld(i,k) >= 1.0e-5_wp - // it may be that the macrophysics has a higher threshold that this - const Real fcld_locutoff = 1.0e-5; - const Real fcld_hicutoff = 0.999; - - // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios (TMRs, - // mol/mol or #/kmol) - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) - // qgcm1, qgcm2, qgcm3 - // qqcwgcm2, qqcwgcm3 - // qaerwatgcm3 ! aerosol water mixing ratios (mol/mol) - - // -------------------------------------------------------------------------------------- - // Determine the number of sub-areas, their fractional areas, and relative - // humidities - // -------------------------------------------------------------------------------------- - // if cloud fraction ~= 0, the grid-cell has a single clear sub-area - // (nsubarea = 1) if cloud fraction ~= 1, the grid-cell has a single cloudy - // sub-area (nsubarea = 1) otherwise, the grid-cell has a - // clear and a cloudy sub-area (nsubarea = 2) - - Real zfcldy = 0; - nsubarea = 0; - ncldy_subarea = 0; - jclea = 0; - jcldy = 0; - - if (cld < fcld_locutoff) { - nsubarea = 1; - jclea = 1; - } else if (cld > fcld_hicutoff) { - zfcldy = 1.0; - nsubarea = 1; - ncldy_subarea = 1; - jcldy = 1; - } else { - zfcldy = cld; - nsubarea = 2; - ncldy_subarea = 1; - jclea = 1; - jcldy = 2; - } - - const Real zfclea = 1.0 - zfcldy; - for (int i = 0; i < maxsubarea(); ++i) - iscldy_subarea[i] = false; - if (jcldy > 0) - iscldy_subarea[jcldy - 1] = true; - for (int i = 0; i < maxsubarea(); ++i) - afracsub[i] = 0.0; - if (jclea > 0) - afracsub[jclea - 1] = zfclea; - if (jcldy > 0) - afracsub[jcldy - 1] = zfcldy; - - // cldy_rh_sameas_clear is just to match mam_refactor. Compiler should - // optimize away. - const int cldy_rh_sameas_clear = 0; - if (ncldy_subarea <= 0) { - for (int i = 0; i < maxsubarea(); ++i) - relhumsub[i] = relhumgcm; - } else if (cldy_rh_sameas_clear > 0) { - for (int i = 0; i < maxsubarea(); ++i) - relhumsub[i] = relhumgcm; - } else { - if (jcldy > 0) { - relhumsub[jcldy - 1] = 1.0; - if (jclea > 0) { - const Real tmpa = - (relhumgcm - afracsub[jcldy - 1]) / afracsub[jclea - 1]; - relhumsub[jclea - 1] = haero::max(0.0, haero::min(1.0, tmpa)); - } - } - } - - // ---------------------------------------------------------------------------- - // Copy grid cell mean mixing ratios. - // These values, together with cloud fraction and a few assumptions, are used - // in the remainder of the subroutine to calculate the sub-area mean mixing - // ratios. - // ---------------------------------------------------------------------------- - // Interstitial aerosols - Real qgcm1[gas_pcnst()], qgcm2[gas_pcnst()], qgcm3[gas_pcnst()]; - for (int i = 0; i < gas_pcnst(); ++i) { - qgcm1[i] = haero::max(0.0, q_pregaschem[i]); - qgcm2[i] = haero::max(0.0, q_precldchem[i]); - qgcm3[i] = haero::max(0.0, q[i]); - } - - // Cloud-borne aerosols - Real qqcwgcm2[gas_pcnst()], qqcwgcm3[gas_pcnst()]; - for (int i = 0; i < gas_pcnst(); ++i) { - qqcwgcm2[i] = haero::max(0.0, qqcw_precldchem[i]); - qqcwgcm3[i] = haero::max(0.0, qqcw[i]); - } - - // aerosol water - Real qaerwatgcm3[num_modes] = {}; - for (int i = 0; i < num_modes; ++i) { - qaerwatgcm3[i] = haero::max(0.0, qaerwat[i]); - } - - // ---------------------------------------------------------------------------- - // Initialize the subarea mean mixing ratios - // ---------------------------------------------------------------------------- - { - const int n = haero::min(maxsubarea(), nsubarea + 1); - for (int i = 0; i < n; ++i) { - for (int j = 0; j < gas_pcnst(); ++j) { - qsub1[j][i] = 0.0; - qsub2[j][i] = 0.0; - qsub3[j][i] = 0.0; - qqcwsub1[j][i] = 0.0; - qqcwsub2[j][i] = 0.0; - qqcwsub3[j][i] = 0.0; - } - for (int j = 0; j < num_modes; ++j) { - qaerwatsub3[j][i] = 0.0; - } - } - } - - // ************************************************************************************************* - // Calculate initial (i.e., before cond/rnam/nnuc/coag) tracer mixing - // ratios within the sub-areas - // - for all-clear or all-cloudy cases, the sub-area TMRs are equal to the - // grid-cell means - // - for partly cloudy case, they are different. This is primarily - // because the - // interstitial aerosol mixing ratios are assumed lower in the cloudy - // sub-area than in the clear sub-area, because much of the aerosol is - // activated in the cloudy sub-area. - // ************************************************************************************************* - // Category I: partly cloudy case - // ************************************************************************************************* - if ((jclea > 0) && (jcldy > 0) && (jclea + jcldy == 3) && (nsubarea == 2)) { - - // --------------------------------------------------------------------- - // Set GAS mixing ratios in sub-areas (for the condensing gases only!!) - // --------------------------------------------------------------------- - for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { - if (lmapcc_all(lmz) == lmapcc_val_gas()) { - - // assume gas in both sub-areas before gas-chem and cloud-chem equal - // grid-cell mean - for (int i = 0; i < nsubarea; ++i) { - qsub1[lmz][i] = qgcm1[lmz]; - qsub2[lmz][i] = qgcm2[lmz]; - } - // assume gas in clear sub-area after cloud-chem equals before - // cloud-chem value - qsub3[lmz][jclea - 1] = qsub2[lmz][jclea - 1]; - // gas in cloud sub-area then determined by grid-cell mean and clear - // values - qsub3[lmz][jcldy - 1] = - (qgcm3[lmz] - zfclea * qsub3[lmz][jclea - 1]) / zfcldy; - - // check that this does not produce a negative value - if (qsub3[lmz][jcldy - 1] < 0.0) { - qsub3[lmz][jcldy - 1] = 0.0; - qsub3[lmz][jclea - 1] = qgcm3[lmz] / zfclea; - } - } - } - // --------------------------------------------------------------------- - // Set CLOUD-BORNE AEROSOL mixing ratios in sub-areas. - // This is straightforward, as the same partitioning factors (0 or 1/f) - // are applied to all mass and number mixing ratios in all modes. - // --------------------------------------------------------------------- - // loop thru log-normal modes - for (int n = 0; n < num_modes; ++n) { - // number - then mass of individual species - of a mode - for (int l2 = -1; l2 < num_species_mode(n); ++l2) { - int lc; - if (l2 == -1) - lc = numptr_amode(n); - else - lc = lmassptr_amode(l2, n); - qqcwsub2[lc][jclea - 1] = 0.0; - qqcwsub2[lc][jcldy - 1] = qqcwgcm2[lc] / zfcldy; - qqcwsub3[lc][jclea - 1] = 0.0; - qqcwsub3[lc][jcldy - 1] = qqcwgcm3[lc] / zfcldy; - } - } - - // --------------------------------------------------------------------- - // Set INTERSTITIAL AEROSOL mixing ratios in sub-areas. - // --------------------------------------------------------------------- - for (int n = 0; n < num_modes; ++n) { - // ------------------------------------- - // Aerosol number - // ------------------------------------- - // grid cell mean, interstitial - Real tmp_q_cellavg_int = qgcm2[numptr_amode(n)]; - // grid cell mean, cloud-borne - Real tmp_q_cellavg_cbn = qqcwgcm2[numptr_amode(n)]; - - Real nmbr_part_fac_clea = 0; - Real nmbr_part_fac_cldy = 0; - subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, - zfclea, nmbr_part_fac_clea, nmbr_part_fac_cldy); - - // Apply the partitioning factors to calculate sub-area mean number - // mixing ratios - - const int la = numptr_amode(n); - - qsub2[la][jclea - 1] = qgcm2[la] * nmbr_part_fac_clea; - qsub2[la][jcldy - 1] = qgcm2[la] * nmbr_part_fac_cldy; - qsub3[la][jclea - 1] = qgcm3[la] * nmbr_part_fac_clea; - qsub3[la][jcldy - 1] = qgcm3[la] * nmbr_part_fac_cldy; - - //------------------------------------- - // Aerosol mass - //------------------------------------- - // For aerosol mass, we use the total grid cell mean - // interstitial/cloud-borne mass mixing ratios to come up with the same - // partitioning for all species in the mode. - - // Compute the total mixing ratios by summing up the individual species - - tmp_q_cellavg_int = 0.0; // grid cell mean, interstitial - tmp_q_cellavg_cbn = 0.0; // grid cell mean, cloud-borne - - for (int l2 = 0; l2 < num_species_mode(n); ++l2) { - tmp_q_cellavg_int += qgcm2[lmassptr_amode(l2, n)]; - tmp_q_cellavg_cbn += qqcwgcm2[lmassptr_amode(l2, n)]; - } - Real mass_part_fac_clea = 0; - Real mass_part_fac_cldy = 0; - // Calculate the partitioning factors - subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, - zfclea, mass_part_fac_clea, mass_part_fac_cldy); - - // Apply the partitioning factors to calculate sub-area mean mass mixing - // ratios - - for (int l2 = 0; l2 < num_species_mode(n); ++l2) { - const int la = lmassptr_amode(l2, n); - - qsub2[la][jclea - 1] = qgcm2[la] * mass_part_fac_clea; - qsub2[la][jcldy - 1] = qgcm2[la] * mass_part_fac_cldy; - qsub3[la][jclea - 1] = qgcm3[la] * mass_part_fac_clea; - qsub3[la][jcldy - 1] = qgcm3[la] * mass_part_fac_cldy; - } - } - - // ************************************************************************************************* - // Category II: all clear, or cld < 1e-5 - // In this case, zfclea=1 and zfcldy=0 - // ************************************************************************************************* - } else if ((jclea == 1) && (jcldy == 0) && (nsubarea == 1)) { - // - // put all the gases and interstitial aerosols in the clear sub-area - // and set mix-ratios = 0 in cloudy sub-area - // for cloud-borne aerosol, do nothing - // because the grid-cell-mean cloud-borne aerosol will be left - // unchanged (i.e., this routine only changes qqcw when cld >= 1e-5) - // - - for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { - if (0 < lmapcc_all(lmz)) { - qsub1[lmz][jclea - 1] = qgcm1[lmz]; - qsub2[lmz][jclea - 1] = qgcm2[lmz]; - qsub3[lmz][jclea - 1] = qgcm3[lmz]; - qqcwsub2[lmz][jclea - 1] = qqcwgcm2[lmz]; - qqcwsub3[lmz][jclea - 1] = qqcwgcm3[lmz]; - } - } - // ************************************************************************************************* - // Category III: all cloudy, or cld > 0.999 - // in this case, zfcldy= and zfclea=0 - // ************************************************************************************************* - } else if ((jclea == 0) && (jcldy == 1) && (nsubarea == 1)) { - // - // put all the gases and interstitial aerosols in the cloudy sub-area - // and set mix-ratios = 0 in clear sub-area - // - for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { - if (0 < lmapcc_all(lmz)) { - qsub1[lmz][jcldy - 1] = qgcm1[lmz]; - qsub2[lmz][jcldy - 1] = qgcm2[lmz]; - qsub3[lmz][jcldy - 1] = qgcm3[lmz]; - qqcwsub2[lmz][jcldy - 1] = qqcwgcm2[lmz]; - qqcwsub3[lmz][jcldy - 1] = qqcwgcm3[lmz]; - } - } - // ************************************************************************************************* - } else { // this should not happen - EKAT_KERNEL_REQUIRE_MSG(false, "*** modal_aero_amicphys - bad jclea, jcldy, nsubarea!"); - } - // ************************************************************************************************* - - // ------------------------------------------------------------------------------------ - // aerosol water -- how to treat this in sub-areas needs more work/thinking - // currently modal_aero_water_uptake calculates qaerwat using - // the grid-cell mean interstital-aerosol mix-rats and the clear-area rh - for (int jsub = 0; jsub < nsubarea; ++jsub) - for (int i = 0; i < num_modes; ++i) - qaerwatsub3[i][jsub] = qaerwatgcm3[i]; - - // ------------------------------------------------------------------------------------ - if (nsubarea == 1) { - // the j=1 subarea is used for some diagnostics - // but is not used in actual calculations - const int j = 1; - for (int i = 0; i < gas_pcnst(); ++i) { - qsub1[i][j] = 0.0; - qsub2[i][j] = 0.0; - qsub3[i][j] = 0.0; - qqcwsub2[i][j] = 0.0; - qqcwsub3[i][j] = 0.0; - } - } -} - -KOKKOS_INLINE_FUNCTION -void mam_amicphys_1subarea_clear( - const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, - const int nsubarea, const bool iscldy_subarea, const Real afracsub, - const Real temp, const Real pmid, const Real pdel, const Real zmid, - const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], - Real dgn_awet[AeroConfig::num_modes()], - Real wetdens[AeroConfig::num_modes()], - const Real qgas1[AeroConfig::num_gas_ids()], - const Real qgas3[AeroConfig::num_gas_ids()], - Real qgas4[AeroConfig::num_gas_ids()], - Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], - const Real qnum3[AeroConfig::num_modes()], - Real qnum4[AeroConfig::num_modes()], - Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], - const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] - [nqtendaa()], - const Real qwtr3[AeroConfig::num_modes()], - Real qwtr4[AeroConfig::num_modes()]) { - static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); - static constexpr int num_modes = AeroConfig::num_modes(); - static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); - - static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); - // Turn off nh3 for now. This is a future enhancement. - static constexpr int igas_nh3 = -999888777; // Same as mam_refactor - static constexpr int iaer_so4 = static_cast(AeroId::SO4); - static constexpr int iaer_pom = static_cast(AeroId::POM); - - const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, - AeroId::None}; - - const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { - {true, true, true, true}, - {true, true, true, true}, - {false, false, false, false}}; - enum { NA, ANAL, IMPL }; - const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; - - // air molar density (kmol/m3) - // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] - const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor - const Real aircon = pmid / (1000 * r_universal * temp); - const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, - 0.58778666490211906, 0.47000362924573563}; - const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; - // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) - // qgas3, qaer3, qnum3 are the current incoming TMRs - // qgas4, qaer4, qnum4 are the updated outgoing TMRs - // - // this routine calculates changes involving - // gas-aerosol exchange (condensation/evaporation) - // growth from smaller to larger modes (renaming) due to condensation - // new particle nucleation - // coagulation - // transfer of particles from hydrophobic modes to hydrophilic modes - // (aging) - // due to condensation and coagulation - // - // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios - // XXX=gas - gas species - // XXX=aer - aerosol mass species (excluding water) - // XXX=wat - aerosol water - // XXX=num - aerosol number - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - current incoming values (before gas-aerosol exchange, newnuc, - // coag) N=4 - updated outgoing values (after gas-aerosol exchange, - // newnuc, coag) - // - // qXXX_delaa are TMR changes (not tendencies) - // for different processes, which are used to produce history output - // for a clear sub-area, the processes are condensation/evaporation (and - // associated aging), renaming, coagulation, and nucleation - - Real qgas_cur[num_gas_ids]; - for (int i = 0; i < num_gas_ids; ++i) - qgas_cur[i] = qgas3[i]; - Real qaer_cur[num_aerosol_ids][num_modes]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_cur[i][j] = qaer3[i][j]; - - Real qnum_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qnum_cur[j] = qnum3[j]; - Real qwtr_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qwtr_cur[j] = qwtr3[j]; - - // qgas_netprod_otrproc = gas net production rate from other processes - // such as gas-phase chemistry and emissions (mol/mol/s) - // this allows the condensation (gasaerexch) routine to apply production and - // condensation loss - // together, which is more accurate numerically - // NOTE - must be >= zero, as numerical method can fail when it is negative - // NOTE - currently only the values for h2so4 and nh3 should be non-zero - Real qgas_netprod_otrproc[num_gas_ids] = {}; - if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { - for (int igas = 0; igas < num_gas_ids; ++igas) { - if (igas == igas_h2so4 || igas == igas_nh3) { - // if config.gaexch_h2so4_uptake_optaa == 2, then - // if qgas increases from pre-gaschem to post-cldchem, - // start from the pre-gaschem mix-ratio and add in the production - // during the integration - // if it decreases, - // start from post-cldchem mix-ratio - // *** currently just do this for h2so4 and nh3 - qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; - if (qgas_netprod_otrproc[igas] >= 0.0) - qgas_cur[igas] = qgas1[igas]; - else - qgas_netprod_otrproc[igas] = 0.0; - } - } - } - Real qgas_del_cond[num_gas_ids] = {}; - Real qgas_del_nnuc[num_gas_ids] = {}; - Real qgas_del_cond_only[num_gas_ids] = {}; - Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; - Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; - Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; - Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; - Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; - Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; - Real qnum_del_cond[num_modes] = {}; - Real qnum_del_rnam[num_modes] = {}; - Real qnum_del_nnuc[num_modes] = {}; - Real qnum_del_coag[num_modes] = {}; - Real qnum_delsub_cond[num_modes] = {}; - Real qnum_delsub_coag[num_modes] = {}; - Real qnum_del_cond_only[num_modes] = {}; - Real dnclusterdt = 0.0; - - const int ntsubstep = 1; - Real dtsubstep = deltat; - if (ntsubstep > 1) - dtsubstep = deltat / ntsubstep; - Real del_h2so4_gasprod = - haero::max(qgas3[igas_h2so4] - qgas1[igas_h2so4], 0.0) / ntsubstep; - - // loop over multiple time sub-steps - for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { - // gas-aerosol exchange - Real uptkrate_h2so4 = 0.0; - Real del_h2so4_aeruptk = 0.0; - Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; - Real qgas_avg[num_gas_ids] = {}; - Real qnum_sv1[num_modes] = {}; - Real qaer_sv1[num_aerosol_ids][num_modes] = {}; - Real qgas_sv1[num_gas_ids] = {}; - - if (config.do_cond) { - - const bool l_calc_gas_uptake_coeff = jtsubstep == 1; - Real uptkaer[num_gas_ids][num_modes] = {}; - - for (int i = 0; i < num_gas_ids; ++i) - qgas_sv1[i] = qgas_cur[i]; - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - - // time sub-step - const Real dtsub_soa_fixed = -1.0; - // Integration order - const int nghq = 2; - const int ntot_soamode = 4; - int niter_out = 0; - Real g0_soa_out = 0; - // gasaerexch::mam_gasaerexch_1subarea( - // nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, - // iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, - // eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, - // aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, - // qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, - // uptkrate_h2so4, niter_out, g0_soa_out); - - if (config.newnuc_h2so4_conc_optaa == 11) - qgas_avg[igas_h2so4] = - 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); - else if (config.newnuc_h2so4_conc_optaa == 12) - qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; - - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_cond[i] += - (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); - - for (int i = 0; i < num_modes; ++i) - qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; - - // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation - // calculations - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_cond_only[i] = qgas_del_cond[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; - for (int i = 0; i < num_modes; ++i) - qnum_del_cond_only[i] = qnum_delsub_cond[i]; - del_h2so4_aeruptk = - qgas_cur[igas_h2so4] - - (qgas_sv1[igas_h2so4] + qgas_netprod_otrproc[igas_h2so4] * dtsubstep); - } else { - for (int i = 0; i < num_gas_ids; ++i) - qgas_avg[i] = qgas_cur[i]; - } - - // renaming after "continuous growth" - if (config.do_rename) { - constexpr int nmodes = AeroConfig::num_modes(); - constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); - const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT - const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; - - Real qnumcw_cur[num_modes] = {}; - Real qaercw_cur[num_aerosol_ids][num_modes] = {}; - Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; - Real mean_std_dev[nmodes]; - Real fmode_dist_tail_fac[nmodes]; - Real v2n_lo_rlx[nmodes]; - Real v2n_hi_rlx[nmodes]; - Real ln_diameter_tail_fac[nmodes]; - int num_pairs = 0; - Real diameter_cutoff[nmodes]; - Real ln_dia_cutoff[nmodes]; - Real diameter_threshold[nmodes]; - Real mass_2_vol[naerosol_species] = {0.15, - 6.4971751412429377e-002, - 0.15, - 7.0588235294117650e-003, - 3.0789473684210526e-002, - 5.1923076923076926e-002, - 156.20986883198000}; - - rename::find_renaming_pairs(dest_mode_of_mode, // in - mean_std_dev, // out - fmode_dist_tail_fac, // out - v2n_lo_rlx, // out - v2n_hi_rlx, // out - ln_diameter_tail_fac, // out - num_pairs, // out - diameter_cutoff, // out - ln_dia_cutoff, diameter_threshold); - - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - Real dgnum_amode[nmodes]; - for (int m = 0; m < nmodes; ++m) { - dgnum_amode[m] = modes(m).nom_diameter; - } - - { - Real qmol_i_cur[num_modes][num_aerosol_ids]; - Real qmol_i_del[num_modes][num_aerosol_ids]; - Real qmol_c_cur[num_modes][num_aerosol_ids]; - Real qmol_c_del[num_modes][num_aerosol_ids]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qmol_i_cur[i][j] = qaer_cur[j][i]; - qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; - qmol_c_cur[i][j] = qaercw_cur[j][i]; - qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; - } - Rename rename; - rename.mam_rename_1subarea_( - iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, - mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, - ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, - diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, - qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); - - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qaer_cur[j][i] = qmol_i_cur[i][j]; - qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; - qaercw_cur[j][i] = qmol_c_cur[i][j]; - qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; - } - } - - for (int i = 0; i < num_modes; ++i) - qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; - } - - // new particle formation (nucleation) - if (config.do_newnuc) { - for (int i = 0; i < num_gas_ids; ++i) - qgas_sv1[i] = qgas_cur[i]; - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - Real qaer_cur_tmp[num_modes][num_aerosol_ids]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qaer_sv1[j][i] = qaer_cur[j][i]; - qaer_cur_tmp[i][j] = qaer_cur[j][i]; - } - Real dnclusterdt_substep = 0; - Real dndt_ait = 0; - Real dmdt_ait = 0; - Real dso4dt_ait = 0; - Real dnh4dt_ait = 0; - Nucleation nucleation; - Nucleation::Config config; - // config.dens_so4a_host = 1770; - config.mw_nh4a_host = 115; - config.mw_so4a_host = 115; - // config.accom_coef_h2so4 = 0.65; - AeroConfig aero_config; - nucleation.init(aero_config, config); - // new version after switching from box model version of nucleation - // compute_tendencies_( - // // Real deltat, - // // Real temp_in, - // // Real press_in, - // // Real zm_in, - // // Real pblh_in, - // // Real relhum, - // // Real uptkrate_h2so4, - // const int nsize, // missing? - // const Real dp_lo_mode, - // const Real dp_hi_mode, - // // const Real qgas_cur[num_gases], - // // const Real qgas_avg[num_gases], - // int &isize_nuc, - // // Real &qnuma_del, - // // Real &qso4a_del, - // // Real &qnh4a_del, - // // Real &qh2so4_del, - // Real &qnh3_del, - // // Real &dnclusterdt - // ) - // old, box-model-based version - // nucleation.compute_tendencies_( - // dtsubstep, temp, pmid, aircon, zmid, pblh, relhum, uptkrate_h2so4, - // del_h2so4_gasprod, del_h2so4_aeruptk, qgas_cur, qgas_avg, qnum_cur, - // qaer_cur_tmp, qwtr_cur, dndt_ait, dmdt_ait, dso4dt_ait, dnh4dt_ait, - // dnclusterdt_substep); - // nucleation.compute_tendencies_( - // dtsubstep, // deltat - // temp, // temp_in - // pmid, // press_in - // aircon, // ? - // zmid, // zm_in - // pblh, // pblh_in - // relhum, // relhum - // uptkrate_h2so4, // uptkrate_h2so4 - // // nsize? - // // dp_lo_mode? - // // dp_hi_mode? - // del_h2so4_aeruptk, // uptkrate_h2so4? - // qgas_cur, // qgas_cur[num_gases]? - // qgas_avg, // qgas_avg[num_gases]? - // // isize_nuc? - // qnum_cur, // qnuma_del? - // qaer_cur_tmp, // ? - // qwtr_cur, // ? - // dndt_ait, // - // dmdt_ait, // - // dso4dt_ait, // qso4a_del - // dnh4dt_ait, // qnh4a_del - // del_h2so4_gasprod, // qh2so4_del? - // // qnh3_del? - // dnclusterdt_substep // dnclusterdt - // ); - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_cur[j][i] = qaer_cur_tmp[i][j]; - - //! Apply the tendencies to the prognostics. - const int nait = static_cast(ModeIndex::Aitken); - qnum_cur[nait] += dndt_ait * dtsubstep; - - if (dso4dt_ait > 0.0) { - static constexpr int iaer_so4 = static_cast(AeroId::SO4); - static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); - - Real delta_q = dso4dt_ait * dtsubstep; - qaer_cur[iaer_so4][nait] += delta_q; - delta_q = haero::min(delta_q, qgas_cur[igas_h2so4]); - qgas_cur[igas_h2so4] -= delta_q; - } - - if (igas_nh3 > 0 && dnh4dt_ait > 0.0) { - static constexpr int iaer_nh4 = - -9999999; // static_cast(AeroId::NH4); - - Real delta_q = dnh4dt_ait * dtsubstep; - qaer_cur[iaer_nh4][nait] += delta_q; - delta_q = haero::min(delta_q, qgas_cur[igas_nh3]); - qgas_cur[igas_nh3] -= delta_q; - } - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_nnuc[i] += (qgas_cur[i] - qgas_sv1[i]); - for (int i = 0; i < num_modes; ++i) - qnum_del_nnuc[i] += (qnum_cur[i] - qnum_sv1[i]); - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_del_nnuc[j][i] += (qaer_cur[j][i] - qaer_sv1[j][i]); - - dnclusterdt = dnclusterdt + dnclusterdt_substep * (dtsubstep / deltat); - } - - // coagulation part - if (config.do_coag) { - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - // coagulation::mam_coag_1subarea(dtsubstep, temp, pmid, aircon, dgn_a, - // dgn_awet, wetdens, qnum_cur, qaer_cur, - // qaer_delsub_coag_in); - for (int i = 0; i < num_modes; ++i) - qnum_delsub_coag[i] = qnum_cur[i] - qnum_sv1[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_delsub_coag[j][i] = qaer_cur[j][i] - qaer_sv1[j][i]; - } - - // primary carbon aging - - aging::mam_pcarbon_aging_1subarea( - dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, - qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); - - // accumulate sub-step q-dels - if (config.do_coag) { - for (int i = 0; i < num_modes; ++i) - qnum_del_coag[i] += qnum_delsub_coag[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_del_coag[j][i] += qaer_delsub_coag[j][i]; - } - if (config.do_cond) { - for (int i = 0; i < num_modes; ++i) - qnum_del_cond[i] += qnum_delsub_cond[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; - } - } - - // final mix ratios - for (int i = 0; i < num_gas_ids; ++i) - qgas4[i] = qgas_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer4[j][i] = qaer_cur[j][i]; - for (int i = 0; i < num_modes; ++i) - qnum4[i] = qnum_cur[i]; - for (int i = 0; i < num_modes; ++i) - qwtr4[i] = qwtr_cur[i]; - - // final mix ratio changes - for (int i = 0; i < num_gas_ids; ++i) { - qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; - qgas_delaa[i][iqtend_rnam()] = 0.0; - qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; - qgas_delaa[i][iqtend_coag()] = 0.0; - qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; - } - for (int i = 0; i < num_modes; ++i) { - qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; - qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; - qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; - qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; - qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; - } - for (int j = 0; j < num_aerosol_ids; ++j) { - for (int i = 0; i < num_modes; ++i) { - qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; - qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; - qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; - qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; - qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; - } - } -} - -KOKKOS_INLINE_FUNCTION -void mam_amicphys_1subarea_cloudy( - const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, - const int nsubarea, const bool iscldy_subarea, const Real afracsub, - const Real temp, const Real pmid, const Real pdel, const Real zmid, - const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], - Real dgn_awet[AeroConfig::num_modes()], - Real wetdens[AeroConfig::num_modes()], - const Real qgas1[AeroConfig::num_gas_ids()], - const Real qgas3[AeroConfig::num_gas_ids()], - Real qgas4[AeroConfig::num_gas_ids()], - Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], - const Real qnum3[AeroConfig::num_modes()], - Real qnum4[AeroConfig::num_modes()], - Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], - const Real qaer2[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], - Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] - [nqtendaa()], - const Real qwtr3[AeroConfig::num_modes()], - Real qwtr4[AeroConfig::num_modes()], - const Real qnumcw3[AeroConfig::num_modes()], - Real qnumcw4[AeroConfig::num_modes()], - Real qnumcw_delaa[AeroConfig::num_modes()][nqqcwtendaa()], - const Real qaercw2[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], - const Real qaercw3[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], - Real qaercw4[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], - Real qaercw_delaa[AeroConfig::num_gas_ids()][AeroConfig::num_modes()] - [nqqcwtendaa()]) { - - // - // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) - // qgas3, qaer3, qaercw3, qnum3, qnumcw3 are the current incoming TMRs - // qgas4, qaer4, qaercw4, qnum4, qnumcw4 are the updated outgoing TMRs - // - // when config.do_cond = false, this routine only calculates changes involving - // growth from smaller to larger modes (renaming) following cloud chemistry - // so gas TMRs are not changed - // when config.do_cond = true, this routine also calculates changes involving - // gas-aerosol exchange (condensation/evaporation) - // transfer of particles from hydrophobic modes to hydrophilic modes - // (aging) - // due to condensation - // currently this routine does not do - // new particle nucleation - because h2so4 gas conc. should be very low in - // cloudy air coagulation - because cloud-borne aerosol would need to be - // included - // - - // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios - // XXX=gas - gas species - // XXX=aer - aerosol mass species (excluding water) - // XXX=wat - aerosol water - // XXX=num - aerosol number - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - current incoming values (before gas-aerosol exchange, newnuc, - // coag) N=4 - updated outgoing values (after gas-aerosol exchange, - // newnuc, coag) - // - // qXXX_delaa are TMR changes (not tendencies) - // for different processes, which are used to produce history output - // for a clear sub-area, the processes are condensation/evaporation (and - // associated aging), - // renaming, coagulation, and nucleation - - // qxxx_del_yyyy are mix-ratio changes over full time step (deltat) - // qxxx_delsub_yyyy are mix-ratio changes over time sub-step (dtsubstep) - - static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); - static constexpr int num_modes = AeroConfig::num_modes(); - static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); - - static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); - // Turn off nh3 for now. This is a future enhancement. - static constexpr int igas_nh3 = -999888777; // Same as mam_refactor - static constexpr int iaer_so4 = static_cast(AeroId::SO4); - static constexpr int iaer_pom = static_cast(AeroId::POM); - - const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, - AeroId::None}; - const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { - {true, true, true, true}, - {true, true, true, true}, - {false, false, false, false}}; - enum { NA, ANAL, IMPL }; - const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; - // air molar density (kmol/m3) - // In order to try to match the results in mam_refactor - // set r_universal as [mJ/(mol)] as in mam_refactor. - // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] - const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor - const Real aircon = pmid / (1000 * r_universal * temp); - const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, - 0.58778666490211906, 0.47000362924573563}; - const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; - - Real qgas_cur[num_gas_ids]; - for (int i = 0; i < num_gas_ids; ++i) - qgas_cur[i] = qgas3[i]; - Real qaer_cur[num_aerosol_ids][num_modes]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_cur[i][j] = qaer3[i][j]; - - Real qnum_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qnum_cur[j] = qnum3[j]; - Real qwtr_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qwtr_cur[j] = qwtr3[j]; - - Real qnumcw_cur[num_modes]; - for (int j = 0; j < num_modes; ++j) - qnumcw_cur[j] = qnumcw3[j]; - - Real qaercw_cur[num_gas_ids][num_modes]; - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_cur[i][j] = qaercw3[i][j]; - - Real qgas_netprod_otrproc[num_gas_ids] = {}; - if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { - for (int igas = 0; igas < num_gas_ids; ++igas) { - if (igas == igas_h2so4 || igas == igas_nh3) { - // if gaexch_h2so4_uptake_optaa == 2, then - // if qgas increases from pre-gaschem to post-cldchem, - // start from the pre-gaschem mix-ratio and add in the production - // during the integration - // if it decreases, - // start from post-cldchem mix-ratio - // *** currently just do this for h2so4 and nh3 - qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; - if (qgas_netprod_otrproc[igas] >= 0.0) - qgas_cur[igas] = qgas1[igas]; - else - qgas_netprod_otrproc[igas] = 0.0; - } - } - } - Real qgas_del_cond[num_gas_ids] = {}; - Real qgas_del_nnuc[num_gas_ids] = {}; - Real qgas_del_cond_only[num_gas_ids] = {}; - Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; - Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; - Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; - Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; - Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; - Real qaercw_del_rnam[num_aerosol_ids][num_modes] = {}; - Real qnum_del_cond[num_modes] = {}; - Real qnum_del_rnam[num_modes] = {}; - Real qnum_del_nnuc[num_modes] = {}; - Real qnum_del_coag[num_modes] = {}; - Real qnum_delsub_cond[num_modes] = {}; - Real qnum_delsub_coag[num_modes] = {}; - Real qnum_del_cond_only[num_modes] = {}; - Real qnumcw_del_rnam[num_modes] = {}; - Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; - - const int ntsubstep = 1; - Real dtsubstep = deltat; - if (ntsubstep > 1) - dtsubstep = deltat / ntsubstep; - - // loop over multiple time sub-steps - - for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { - // gas-aerosol exchange - Real uptkrate_h2so4 = 0.0; - Real qgas_avg[num_gas_ids] = {}; - Real qgas_sv1[num_gas_ids] = {}; - Real qnum_sv1[num_modes] = {}; - Real qaer_sv1[num_aerosol_ids][num_modes] = {}; - Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; - - if (config.do_cond) { - - const bool l_calc_gas_uptake_coeff = jtsubstep == 1; - Real uptkaer[num_gas_ids][num_modes] = {}; - - for (int i = 0; i < num_gas_ids; ++i) - qgas_sv1[i] = qgas_cur[i]; - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - - const int nghq = 2; - const int ntot_soamode = 4; - int niter_out = 0; - Real g0_soa_out = 0; - // time sub-step - const Real dtsub_soa_fixed = -1.0; - // gasaerexch::mam_gasaerexch_1subarea( - // nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, - // iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, - // eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, - // aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, - // qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, - // uptkrate_h2so4, niter_out, g0_soa_out); - - if (config.newnuc_h2so4_conc_optaa == 11) - qgas_avg[igas_h2so4] = - 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); - else if (config.newnuc_h2so4_conc_optaa == 12) - qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; - - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_cond[i] += - (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); - - for (int i = 0; i < num_modes; ++i) - qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; - - // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation - // calculations - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; - for (int i = 0; i < num_gas_ids; ++i) - qgas_del_cond_only[i] = qgas_del_cond[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; - for (int i = 0; i < num_modes; ++i) - qnum_del_cond_only[i] = qnum_delsub_cond[i]; - - } else { - for (int i = 0; i < num_gas_ids; ++i) - qgas_avg[i] = qgas_cur[i]; - } - // renaming after "continuous growth" - if (config.do_rename) { - constexpr int nmodes = AeroConfig::num_modes(); - constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); - const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT - const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; - - Real qnumcw_cur[num_modes] = {}; - Real qaercw_cur[num_aerosol_ids][num_modes] = {}; - Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; - Real mean_std_dev[nmodes]; - Real fmode_dist_tail_fac[nmodes]; - Real v2n_lo_rlx[nmodes]; - Real v2n_hi_rlx[nmodes]; - Real ln_diameter_tail_fac[nmodes]; - int num_pairs = 0; - Real diameter_cutoff[nmodes]; - Real ln_dia_cutoff[nmodes]; - Real diameter_threshold[nmodes]; - Real mass_2_vol[naerosol_species] = {0.15, - 6.4971751412429377e-002, - 0.15, - 7.0588235294117650e-003, - 3.0789473684210526e-002, - 5.1923076923076926e-002, - 156.20986883198000}; - - rename::find_renaming_pairs(dest_mode_of_mode, // in - mean_std_dev, // out - fmode_dist_tail_fac, // out - v2n_lo_rlx, // out - v2n_hi_rlx, // out - ln_diameter_tail_fac, // out - num_pairs, // out - diameter_cutoff, // out - ln_dia_cutoff, diameter_threshold); - - for (int i = 0; i < num_modes; ++i) - qnum_sv1[i] = qnum_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_sv1[j][i] = qaer_cur[j][i]; - Real dgnum_amode[nmodes]; - for (int m = 0; m < nmodes; ++m) { - dgnum_amode[m] = modes(m).nom_diameter; - } - - // qaercw_delsub_grow4rnam = change in qaercw from cloud chemistry - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_delsub_grow4rnam[i][j] = - (qaercw3[i][j] - qaercw2[i][j]) / ntsubstep; - Real qnumcw_sv1[num_modes]; - for (int i = 0; i < num_modes; ++i) - qnumcw_sv1[i] = qnumcw_cur[i]; - Real qaercw_sv1[num_aerosol_ids][num_modes]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_sv1[i][j] = qaercw_cur[i][j]; - - { - Real qmol_i_cur[num_modes][num_aerosol_ids]; - Real qmol_i_del[num_modes][num_aerosol_ids]; - Real qmol_c_cur[num_modes][num_aerosol_ids]; - Real qmol_c_del[num_modes][num_aerosol_ids]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qmol_i_cur[i][j] = qaer_cur[j][i]; - qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; - qmol_c_cur[i][j] = qaercw_cur[j][i]; - qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; - } - - Rename rename; - rename.mam_rename_1subarea_( - iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, - mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, - ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, - diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, - qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); - - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) { - qaer_cur[j][i] = qmol_i_cur[i][j]; - qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; - qaercw_cur[j][i] = qmol_c_cur[i][j]; - qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; - } - } - for (int i = 0; i < num_modes; ++i) - qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; - for (int i = 0; i < num_modes; ++i) - qnumcw_del_rnam[i] += qnumcw_cur[i] - qnumcw_sv1[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_del_rnam[i][j] += qaercw_cur[i][j] - qaercw_sv1[i][j]; - } - - // primary carbon aging - if (config.do_cond) { - aging::mam_pcarbon_aging_1subarea( - dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, - qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); - } - // accumulate sub-step q-dels - if (config.do_cond) { - for (int i = 0; i < num_modes; ++i) - qnum_del_cond[i] += qnum_delsub_cond[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; - } - } - // final mix ratios - for (int i = 0; i < num_gas_ids; ++i) - qgas4[i] = qgas_cur[i]; - for (int j = 0; j < num_aerosol_ids; ++j) - for (int i = 0; i < num_modes; ++i) - qaer4[j][i] = qaer_cur[j][i]; - for (int i = 0; i < num_modes; ++i) - qnum4[i] = qnum_cur[i]; - for (int i = 0; i < num_modes; ++i) - qwtr4[i] = qwtr_cur[i]; - for (int i = 0; i < num_modes; ++i) - qnumcw4[i] = qnumcw_cur[i]; - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw4[i][j] = qaercw_cur[i][j]; - - // final mix ratio changes - for (int i = 0; i < num_gas_ids; ++i) { - qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; - qgas_delaa[i][iqtend_rnam()] = 0.0; - qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; - qgas_delaa[i][iqtend_coag()] = 0.0; - qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; - } - for (int i = 0; i < num_modes; ++i) { - qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; - qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; - qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; - qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; - qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; - } - for (int j = 0; j < num_aerosol_ids; ++j) { - for (int i = 0; i < num_modes; ++i) { - qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; - qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; - qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; - qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; - qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; - } - } - for (int i = 0; i < num_modes; ++i) - qnumcw_delaa[i][iqqcwtend_rnam()] = qnumcw_del_rnam[i]; - for (int i = 0; i < num_aerosol_ids; ++i) - for (int j = 0; j < num_modes; ++j) - qaercw_delaa[i][j][iqqcwtend_rnam()] = qaercw_del_rnam[i][j]; -} - -KOKKOS_INLINE_FUNCTION -void mam_amicphys_1gridcell( - const AmicPhysConfig& config, const int nstep, const Real deltat, const int nsubarea, - const int ncldy_subarea, const bool iscldy_subarea[maxsubarea()], - const Real afracsub[maxsubarea()], const Real temp, const Real pmid, - const Real pdel, const Real zmid, const Real pblh, - const Real relhumsub[maxsubarea()], Real dgn_a[AeroConfig::num_modes()], - Real dgn_awet[AeroConfig::num_modes()], - Real wetdens[AeroConfig::num_modes()], - const Real qsub1[AeroConfig::num_gas_ids()][maxsubarea()], - const Real qsub2[AeroConfig::num_gas_ids()][maxsubarea()], - const Real qqcwsub2[AeroConfig::num_gas_ids()][maxsubarea()], - const Real qsub3[AeroConfig::num_gas_ids()][maxsubarea()], - const Real qqcwsub3[AeroConfig::num_gas_ids()][maxsubarea()], - Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()], - Real qsub4[AeroConfig::num_gas_ids()][maxsubarea()], - Real qqcwsub4[AeroConfig::num_gas_ids()][maxsubarea()], - Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()], - Real qsub_tendaa[AeroConfig::num_gas_ids()][nqtendaa()][maxsubarea()], - Real qqcwsub_tendaa[AeroConfig::num_gas_ids()][nqqcwtendaa()][maxsubarea()]) { - - // - // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) - // qsub3 and qqcwsub3 are the incoming current TMRs - // qsub4 and qqcwsub4 are the outgoing updated TMRs - // - // qsubN and qqcwsubN (N=1:4) are tracer mixing ratios (TMRs, mol/mol or - // #/kmol) in sub-areas - // currently there are just clear and cloudy sub-areas - // the N=1:4 have same meanings as for qgcmN - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) - // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) - // qsub_tendaa and qqcwsub_tendaa are TMR tendencies - // for different processes, which are used to produce history output - // the processes are condensation/evaporation (and associated aging), - // renaming, coagulation, and nucleation - - static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); - static constexpr int num_modes = AeroConfig::num_modes(); - static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); - - // the q--4 values will be equal to q--3 values unless they get changed - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < maxsubarea(); ++j) { - qsub4[i][j] = qsub3[i][j]; - qqcwsub4[i][j] = qqcwsub3[i][j]; - } - for (int i = 0; i < num_modes; ++i) - for (int j = 0; j < maxsubarea(); ++j) - qaerwatsub4[i][j] = qaerwatsub3[i][j]; - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < nqtendaa(); ++j) - for (int k = 0; k < maxsubarea(); ++k) - qsub_tendaa[i][j][k] = 0; - for (int i = 0; i < num_gas_ids; ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - for (int k = 0; k < maxsubarea(); ++k) - qqcwsub_tendaa[i][j][k] = 0.0; - - for (int jsub = 0; jsub < nsubarea; ++jsub) { - AmicPhysConfig sub_config = config; - if (iscldy_subarea[jsub]) { - sub_config.do_cond = config.do_cond; - sub_config.do_rename = config.do_rename; - sub_config.do_newnuc = false; - sub_config.do_coag = false; - } - const bool do_map_gas_sub = sub_config.do_cond || sub_config.do_newnuc; - - // map incoming sub-area mix-ratios to gas/aer/num arrays - Real qgas1[num_gas_ids] = {}; - Real qgas3[num_gas_ids] = {}; - Real qgas4[num_gas_ids] = {}; - if (do_map_gas_sub) { - // for cldy subarea, only do gases if doing gaexch - for (int igas = 0; igas < 2; ++igas) { - const int l = lmap_gas(igas); - qgas1[igas] = qsub1[l][jsub] * fcvt_gas(igas); - qgas3[igas] = qsub3[l][jsub] * fcvt_gas(igas); - qgas4[igas] = qgas3[igas]; - } - } - Real qaer2[num_aerosol_ids][num_modes] = {}; - Real qaer3[num_aerosol_ids][num_modes] = {}; - Real qnum3[num_modes] = {}; - Real qaer4[num_aerosol_ids][num_modes] = {}; - Real qnum4[num_modes] = {}; - Real qwtr3[num_modes] = {}; - Real qwtr4[num_modes] = {}; - for (int n = 0; n < num_modes; ++n) { - qnum3[n] = qsub3[n][jsub] * fcvt_num(); - qnum4[n] = qnum3[n]; - for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { - qaer2[iaer][n] = qsub2[iaer][jsub] * fcvt_aer(iaer); - qaer3[iaer][n] = qsub3[iaer][jsub] * fcvt_aer(iaer); - qaer4[iaer][n] = qaer3[iaer][n]; - } - qwtr3[n] = qaerwatsub3[n][jsub] * fcvt_wtr(); - qwtr4[n] = qwtr3[n]; - } - Real qaercw2[num_aerosol_ids][num_modes] = {}; - Real qaercw3[num_aerosol_ids][num_modes] = {}; - Real qnumcw3[num_modes] = {}; - Real qaercw4[num_aerosol_ids][num_modes] = {}; - Real qnumcw4[num_modes] = {}; - if (iscldy_subarea[jsub]) { - // only do cloud-borne for cloudy - for (int n = 0; n < num_modes; ++n) { - qnumcw3[n] = qqcwsub3[n][jsub] * fcvt_num(); - qnumcw4[n] = qnumcw3[n]; - for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { - qaercw2[iaer][n] = qqcwsub2[n][jsub] * fcvt_aer(iaer); - qaercw3[iaer][n] = qqcwsub3[n][jsub] * fcvt_aer(iaer); - qaercw4[iaer][n] = qaercw3[iaer][n]; - } - } - } - - Real qgas_delaa[num_gas_ids][nqtendaa()] = {}; - Real qnum_delaa[num_modes][nqtendaa()] = {}; - Real qnumcw_delaa[num_modes][nqqcwtendaa()] = {}; - Real qaer_delaa[num_aerosol_ids][num_modes][nqtendaa()] = {}; - Real qaercw_delaa[num_aerosol_ids][num_modes][nqqcwtendaa()] = {}; - - if (iscldy_subarea[jsub]) { - mam_amicphys_1subarea_cloudy(sub_config, nstep, deltat, - jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, - pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, - qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer2, qaer3, - qaer4, qaer_delaa, qwtr3, qwtr4, qnumcw3, qnumcw4, qnumcw_delaa, - qaercw2, qaercw3, qaercw4, qaercw_delaa); - } else { - mam_amicphys_1subarea_clear(sub_config, nstep, deltat, - jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, - pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, - qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer3, qaer4, - qaer_delaa, qwtr3, qwtr4); - // map gas/aer/num arrays (mix-ratio and del=change) back to sub-area - // arrays - - if (do_map_gas_sub) { - for (int igas = 0; igas < 2; ++igas) { - const int l = lmap_gas(igas); - qsub4[l][jsub] = qgas4[igas] / fcvt_gas(igas); - for (int i = 0; i < nqtendaa(); ++i) - qsub_tendaa[l][i][jsub] = - qgas_delaa[igas][i] / (fcvt_gas(igas) * deltat); - } - } - for (int n = 0; n < num_modes; ++n) { - qsub4[n][jsub] = qnum4[n] / fcvt_num(); - for (int i = 0; i < nqtendaa(); ++i) - qsub_tendaa[n][i][jsub] = qnum_delaa[n][i] / (fcvt_num() * deltat); - for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { - qsub4[iaer][jsub] = qaer4[iaer][n] / fcvt_aer(iaer); - for (int i = 0; i < nqtendaa(); ++i) - qsub_tendaa[iaer][i][jsub] = - qaer_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); - } - qaerwatsub4[n][jsub] = qwtr4[n] / fcvt_wtr(); - - if (iscldy_subarea[jsub]) { - qqcwsub4[n][jsub] = qnumcw4[n] / fcvt_num(); - for (int i = 0; i < nqqcwtendaa(); ++i) - qqcwsub_tendaa[n][i][jsub] = - qnumcw_delaa[n][i] / (fcvt_num() * deltat); - for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { - qqcwsub4[iaer][jsub] = qaercw4[iaer][n] / fcvt_aer(iaer); - for (int i = 0; i < nqqcwtendaa(); ++i) - qqcwsub_tendaa[iaer][i][jsub] = - qaercw_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); - } - } - } - } - } -} - -} // anonymous namespace - -KOKKOS_INLINE_FUNCTION -void modal_aero_amicphys_intr( - const AmicPhysConfig& config, const int nstep, const Real deltat, const Real t, const Real pmid, const Real pdel, - const Real zm, const Real pblh, const Real qv, const Real cld, - Real q[gas_pcnst()], Real qqcw[gas_pcnst()], const Real q_pregaschem[gas_pcnst()], - const Real q_precldchem[gas_pcnst()], const Real qqcw_precldchem[gas_pcnst()], - Real q_tendbb[gas_pcnst()][nqtendbb()], Real qqcw_tendbb[gas_pcnst()][nqtendbb()], - Real dgncur_a[AeroConfig::num_modes()], - Real dgncur_awet[AeroConfig::num_modes()], - Real wetdens_host[AeroConfig::num_modes()], - Real qaerwat[AeroConfig::num_modes()]) { - - /* - nstep ! model time-step number - nqtendbb ! dimension for q_tendbb - nqqcwtendbb ! dimension f - deltat ! - q(ncol,pver,pcnstxx) ! current tracer mixing ratios (TMRs) - these values are updated (so out /= in) - *** MUST BE #/kmol-air for number - *** MUST BE mol/mol-air for mass - *** NOTE ncol dimension - qqcw(ncol,pver,pcnstxx) - like q but for cloud-borner tracers - these values are updated - q_pregaschem(ncol,pver,pcnstxx) ! q TMRs before gas-phase - chemistry q_precldchem(ncol,pver,pcnstxx) ! q TMRs before cloud - chemistry qqcw_precldchem(ncol,pver,pcnstxx) ! qqcw TMRs before cloud - chemistry q_tendbb(ncol,pver,pcnstxx,nqtendbb()) ! TMR tendencies for - box-model diagnostic output qqcw_tendbb(ncol,pver,pcnstx t(pcols,pver) ! - temperature at model levels (K) pmid(pcols,pver) ! pressure at model - level centers (Pa) pdel(pcols,pver) ! pressure thickness of levels - (Pa) zm(pcols,pver) ! altitude (above ground) at level centers (m) - pblh(pcols) ! planetary boundary layer depth (m) - qv(pcols,pver) ! specific humidity (kg/kg) - cld(ncol,pver) ! cloud fraction (-) *** NOTE ncol dimension - dgncur_a(pcols,pver,ntot_amode) - dgncur_awet(pcols,pver,ntot_amode) - ! dry & wet geo. mean dia. (m) of - number distrib. wetdens_host(pcols,pver,ntot_amode) ! interstitial - aerosol wet density (kg/m3) - - qaerwat(pcols,pver,ntot_amode aerosol water mixing ratio (kg/kg, - NOT mol/mol) - - */ - - // !DESCRIPTION: - // calculates changes to gas and aerosol TMRs (tracer mixing ratios) from - // gas-aerosol exchange (condensation/evaporation) - // growth from smaller to larger modes (renaming) due to both - // condensation and cloud chemistry - // new particle nucleation - // coagulation - // transfer of particles from hydrophobic modes to hydrophilic modes - // (aging) - // due to condensation and coagulation - // - // the incoming mixing ratios (q and qqcw) are updated before output - // - // !REVISION HISTORY: - // RCE 07.04.13: Adapted from earlier version of CAM5 modal aerosol - // routines - // for these processes - // - - static constexpr int num_modes = AeroConfig::num_modes(); - - // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios - // (TMRs, mol/mol or #/kmol) - // N=1 - before gas-phase chemistry - // N=2 - before cloud chemistry - // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) - // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) - - // qsubN and qqcwsubN (N=1:4) are TMRs in sub-areas - // currently there are just clear and cloudy sub-areas - // the N=1:4 have same meanings as for qgcmN - - // q_coltendaa and qqcw_coltendaa are column-integrated tendencies - // for different processes, which are output to history - // the processes are condensation/evaporation (and associated aging), - // renaming, coagulation, and nucleation - - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqtendbb(); ++j) - q_tendbb[i][j] = 0.0, qqcw_tendbb[i][j] = 0.0; - - // get saturation mixing ratio - // call qsat( t(1:ncol,1:pver), pmid(1:ncol,1:pvnner), & - // ev_sat(1:ncol,1:pver), qv_sat(1:ncol,1:pver) ) - const Real epsqs = haero::Constants::weight_ratio_h2o_air; - // Saturation vapor pressure - const Real ev_sat = conversions::vapor_saturation_pressure_magnus(t, pmid); - // Saturation specific humidity - const Real qv_sat = epsqs * ev_sat / (pmid - (1 - epsqs) * ev_sat); - - const Real relhumgcm = haero::max(0.0, haero::min(1.0, qv / qv_sat)); - - // Set up cloudy/clear subareas inside a grid cell - int nsubarea, ncldy_subarea, jclea, jcldy; - bool iscldy_subarea[maxsubarea()]; - Real afracsub[maxsubarea()]; - Real relhumsub[maxsubarea()]; - Real qsub1[gas_pcnst()][maxsubarea()]; - Real qsub2[gas_pcnst()][maxsubarea()]; - Real qsub3[gas_pcnst()][maxsubarea()]; - Real qqcwsub1[gas_pcnst()][maxsubarea()]; - Real qqcwsub2[gas_pcnst()][maxsubarea()]; - Real qqcwsub3[gas_pcnst()][maxsubarea()]; - // aerosol water mixing ratios (mol/mol) - Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()]; - construct_subareas_1gridcell(cld, relhumgcm, // in - q_pregaschem, q_precldchem, // in - qqcw_precldchem, // in - q, qqcw, // in - nsubarea, ncldy_subarea, jclea, jcldy, // out - iscldy_subarea, afracsub, relhumsub, // out - qsub1, qsub2, qsub3, // out - qqcwsub1, qqcwsub2, qqcwsub3, qaerwatsub3, // out - qaerwat // in - ); - - // Initialize the "after-amicphys" values - Real qsub4[gas_pcnst()][maxsubarea()] = {}; - Real qqcwsub4[gas_pcnst()][maxsubarea()] = {}; - Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()] = {}; - - // - // start integration - // - Real dgn_a[num_modes], dgn_awet[num_modes], wetdens[num_modes]; - for (int n = 0; n < num_modes; ++n) { - dgn_a[n] = dgncur_a[n]; - dgn_awet[n] = dgncur_awet[n]; - wetdens[n] = haero::max(1000.0, wetdens_host[n]); - } - Real qsub_tendaa[gas_pcnst()][nqtendaa()][maxsubarea()] = {}; - Real qqcwsub_tendaa[gas_pcnst()][nqqcwtendaa()][maxsubarea()] = {}; - mam_amicphys_1gridcell(config, nstep, deltat, - nsubarea, ncldy_subarea, iscldy_subarea, afracsub, t, - pmid, pdel, zm, pblh, relhumsub, dgn_a, dgn_awet, - wetdens, qsub1, qsub2, qqcwsub2, qsub3, qqcwsub3, - qaerwatsub3, qsub4, qqcwsub4, qaerwatsub4, qsub_tendaa, - qqcwsub_tendaa); - - // - // form new grid-mean mix-ratios - Real qgcm4[gas_pcnst()]; - Real qgcm_tendaa[gas_pcnst()][nqtendaa()]; - Real qaerwatgcm4[num_modes]; - if (nsubarea == 1) { - for (int i = 0; i < gas_pcnst(); ++i) - qgcm4[i] = qsub4[i][0]; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqtendaa(); ++j) - qgcm_tendaa[i][j] = qsub_tendaa[i][j][0]; - for (int i = 0; i < num_modes; ++i) - qaerwatgcm4[i] = qaerwatsub4[i][0]; - } else { - for (int i = 0; i < gas_pcnst(); ++i) - qgcm4[i] = 0.0; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqtendaa(); ++j) - qgcm_tendaa[i][j] = 0.0; - for (int n = 0; n < nsubarea; ++n) { - for (int i = 0; i < gas_pcnst(); ++i) - qgcm4[i] += qsub4[i][n] * afracsub[n]; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqtendaa(); ++j) - qgcm_tendaa[i][j] = - qgcm_tendaa[i][j] + qsub_tendaa[i][j][n] * afracsub[n]; - } - for (int i = 0; i < num_modes; ++i) - // for aerosol water use the clear sub-area value - qaerwatgcm4[i] = qaerwatsub4[i][jclea - 1]; - } - Real qqcwgcm4[gas_pcnst()]; - Real qqcwgcm_tendaa[gas_pcnst()][nqqcwtendaa()]; - if (ncldy_subarea <= 0) { - for (int i = 0; i < gas_pcnst(); ++i) - qqcwgcm4[i] = haero::max(0.0, qqcw[i]); - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - qqcwgcm_tendaa[i][j] = 0.0; - } else if (nsubarea == 1) { - for (int i = 0; i < gas_pcnst(); ++i) - qqcwgcm4[i] = qqcwsub4[i][0]; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - qqcwgcm_tendaa[i][j] = qqcwsub_tendaa[i][j][0]; - } else { - for (int i = 0; i < gas_pcnst(); ++i) - qqcwgcm4[i] = 0.0; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - qqcwgcm_tendaa[i][j] = 0.0; - for (int n = 0; n < nsubarea; ++n) { - if (iscldy_subarea[n]) { - for (int i = 0; i < gas_pcnst(); ++i) - qqcwgcm4[i] += qqcwsub4[i][n] * afracsub[n]; - for (int i = 0; i < gas_pcnst(); ++i) - for (int j = 0; j < nqqcwtendaa(); ++j) - qqcwgcm_tendaa[i][j] += qqcwsub_tendaa[i][j][n] * afracsub[n]; - } - } - } - - for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { - if (lmapcc_all(lmz) > 0) { - // HW, to ensure non-negative - q[lmz] = haero::max(qgcm4[lmz], 0.0); - if (lmapcc_all(lmz) >= lmapcc_val_aer()) { - // HW, to ensure non-negative - qqcw[lmz] = haero::max(qqcwgcm4[lmz], 0.0); - } - } - } - for (int i = 0; i < gas_pcnst(); ++i) { - if (iqtend_cond() < nqtendbb()) - q_tendbb[i][iqtend_cond()] = qgcm_tendaa[i][iqtend_cond()]; - if (iqtend_rnam() < nqtendbb()) - q_tendbb[i][iqtend_rnam()] = qgcm_tendaa[i][iqtend_rnam()]; - if (iqtend_nnuc() < nqtendbb()) - q_tendbb[i][iqtend_nnuc()] = qgcm_tendaa[i][iqtend_nnuc()]; - if (iqtend_coag() < nqtendbb()) - q_tendbb[i][iqtend_coag()] = qgcm_tendaa[i][iqtend_coag()]; - if (iqqcwtend_rnam() < nqqcwtendbb()) - qqcw_tendbb[i][iqqcwtend_rnam()] = qqcwgcm_tendaa[i][iqqcwtend_rnam()]; - } - for (int i = 0; i < num_modes; ++i) - qaerwat[i] = qaerwatgcm4[i]; -} - -} // namespace scream::impl From e715b8b471a5173c1efc3668f54b6056a2884e47 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Mon, 4 Nov 2024 17:33:17 -0700 Subject: [PATCH 348/366] update mam4xx submodule to fix gpu compilation error --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 59990d402f4..d1c70ab972b 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 59990d402f4115ce3a36a8c00eb4d059e703a583 +Subproject commit d1c70ab972bc00269939fd3296d93a719f273105 From cc63e5d67ed616b06a58514a44f3326322438bfe Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 4 Nov 2024 20:24:40 -0800 Subject: [PATCH 349/366] EAMxx: Fixes a gpu bug requiring a particular format for dstflx --- components/eamxx/tests/single-process/mam/emissions/input.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index a4564cb3949..0819562d959 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -43,7 +43,7 @@ initial_conditions: # we should get the following variables from other processes pbl_height : 1.0 - dstflx : [ 0.00000000000000000E+000, 0.00000000000000000E+000, 0.00000000000000000E+000, 0.00000000000000000E+000] + dstflx : 0.0 ocnfrac: 0.10000000000000000E+001 sst: 0.30178553874977507E+003 cldfrac_tot: 0.138584624960092 From 40a6190f5557002f1429c36a4249663fc378a1a7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 4 Nov 2024 20:39:44 -0800 Subject: [PATCH 350/366] EAMxx: Clang format --- ...mxx_mam_microphysics_process_interface.cpp | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 57eaf927548..328afc89690 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -2,9 +2,13 @@ // impl namespace for some driver level functions for microphysics +<<<<<<< HEAD #include "readfiles/photo_table_utils.cpp" #include "readfiles/find_season_index_utils.hpp" +======= +>>>>>>> EAMxx: Clang format #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" +#include "readfiles/photo_table_utils.cpp" namespace scream { @@ -36,7 +40,7 @@ MAMMicrophysics::MAMMicrophysics(const ekat::Comm &comm, config_.linoz.o3_lbl = m_params.get("mam4_o3_lbl"); config_.linoz.o3_tau = m_params.get("mam4_o3_tau"); config_.linoz.o3_sfc = m_params.get("mam4_o3_sfc"); - config_.linoz.psc_T = m_params.get("mam4_psc_T"); + config_.linoz.psc_T = m_params.get("mam4_psc_T"); } AtmosphereProcessType MAMMicrophysics::type() const { @@ -81,19 +85,20 @@ void MAMMicrophysics::set_grids( // ----------- Atmospheric quantities ------------- // Specific humidity [kg/kg](Require only for building DS) - add_tracer("qv", grid_, kg/kg); // specific humidity + add_tracer("qv", grid_, kg / kg); // specific humidity // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) - add_tracer("qc", grid_, kg/kg); // cloud liquid wet mixing ratio + add_tracer("qc", grid_, kg / kg); // cloud liquid wet mixing ratio // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) - add_tracer("qi", grid_, kg/kg); // ice wet mixing ratio + add_tracer("qi", grid_, kg / kg); // ice wet mixing ratio // Cloud liquid number mixing ratio [1/kg](Require only for building DS) - add_tracer("nc", grid_, n_unit); // cloud liquid wet number mixing ratio + add_tracer("nc", grid_, + n_unit); // cloud liquid wet number mixing ratio // Cloud ice number mixing ratio [1/kg](Require only for building DS) - add_tracer("ni", grid_, n_unit); // ice number mixing ratio + add_tracer("ni", grid_, n_unit); // ice number mixing ratio // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); @@ -161,7 +166,7 @@ void MAMMicrophysics::set_grids( mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - add_tracer(int_mmr_field_name, grid_, kg/kg); + add_tracer(int_mmr_field_name, grid_, kg / kg); } } // for loop species } // for loop nmodes interstitial @@ -183,7 +188,7 @@ void MAMMicrophysics::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_tracer(gas_mmr_field_name, grid_, kg/kg); + add_tracer(gas_mmr_field_name, grid_, kg / kg); } // Creating a Linoz reader and setting Linoz parameters involves reading data @@ -715,10 +720,17 @@ void MAMMicrophysics::run_impl(const double dt) { const auto zenith_angle = acos_cosine_zenith_; constexpr int gas_pcnst = mam_coupling::gas_pcnst(); +<<<<<<< HEAD const auto& elevated_emis_output = elevated_emis_output_; const auto& extfrc = extfrc_; const auto& forcings = forcings_; constexpr int extcnt = mam4::gas_chemistry::extcnt; +======= + const auto &vert_emis_output = vert_emis_output_; + const auto &extfrc = extfrc_; + const auto &forcings = forcings_; + constexpr int extcnt = mam4::gas_chemistry::extcnt; +>>>>>>> EAMxx: Clang format const int offset_aerosol = mam4::utils::gasses_start_ind(); Real adv_mass_kg_per_moles[gas_pcnst]; @@ -809,9 +821,9 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_o3col_clim_icol, linoz_PmL_clim_icol, linoz_dPmL_dO3_icol, linoz_dPmL_dT_icol, linoz_dPmL_dO3col_icol, linoz_cariolle_pscs_icol, eccf, adv_mass_kg_per_moles, clsmap_4, - permute_4, offset_aerosol, - config.linoz.o3_sfc, config.linoz.o3_tau, config.linoz.o3_lbl, - dry_diameter_icol, wet_diameter_icol, wetdens_icol); + permute_4, offset_aerosol, config.linoz.o3_sfc, config.linoz.o3_tau, + config.linoz.o3_lbl, dry_diameter_icol, wet_diameter_icol, + wetdens_icol); }); // parallel_for for the column loop Kokkos::fence(); From f3a29b109af2891bb212d233cca88e3880d1bdf2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 4 Nov 2024 20:50:43 -0800 Subject: [PATCH 351/366] EAMxx: Fixed some comments paths and other minor cleanup --- .../eamxx/cime_config/namelist_defaults_scream.xml | 2 +- ...m_srf_and_online_emissions_process_interface.cpp | 10 +++++----- ...m_srf_and_online_emissions_process_interface.hpp | 1 + .../src/physics/mam/readfiles/marine_organics.hpp | 10 ---------- .../physics/mam/readfiles/marine_organics_impl.hpp | 8 -------- .../src/physics/mam/readfiles/soil_erodibility.hpp | 13 ------------- .../physics/mam/readfiles/soil_erodibility_impl.hpp | 6 ------ 7 files changed, 7 insertions(+), 43 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index fd62cdc01e3..9c11c76aea1 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -376,7 +376,7 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/dst_ne30pg2_c20241028.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/dst_ne4pg2_c20241028.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/dst_ne4pg2_c20241028.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne30pg2_c20241030.nc diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 190dc1da27d..a3efea7d32d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -15,7 +15,7 @@ using soilErodibilityFunc = MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { - // FIXME: Do we want to read dust emiss factor hfrom namelist?? + // FIXME: Do we want to read dust emiss factor from the namelist?? /* Anything that can be initialized without grid information can be * initialized here. Like universal constants. */ @@ -238,7 +238,7 @@ void MAMSrfOnlineEmiss::set_grids( } // srf emissions file read init // ------------------------------------------------------------- - // setup to enable reading soil erodibility file + // Setup to enable reading soil erodibility file // ------------------------------------------------------------- const std::string soil_erodibility_data_file = @@ -257,12 +257,12 @@ void MAMSrfOnlineEmiss::set_grids( serod_dataReader_); // output // ------------------------------------------------------------- - // setup to enable reading marine organics file + // Setup to enable reading marine organics file // ------------------------------------------------------------- const std::string marine_organics_data_file = m_params.get("marine_organics_file"); - // Field to be read from file (order matters as they are read in the same + // Fields to be read from file (order matters as they are read in the same // order) const std::vector marine_org_fld_name = { "TRUEPOLYC", "TRUEPROTC", "TRUELIPC"}; @@ -477,7 +477,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Output view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); - // Comput online emissions + // Compute online emissions // NOTE: mam4::aero_model_emissions calculates mass and number emission // fluxes in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert mam4::aero_model_emissions::aero_model_emissions( diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index b1cba682bb6..ae00098b079 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -130,6 +130,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { const int gas_start_ind = mam4::utils::gasses_start_ind(); // FIXME: Is there a better way to zero out a select indices? + // We should probably do it in run_impl directly for(int ispc = gas_start_ind; ispc < pcnst; ++ispc) { flux_col(ispc) = 0; } diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp index 92136d315f5..a04dff129f4 100644 --- a/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp @@ -15,7 +15,6 @@ struct marineOrganicsFunctions { using MemberType = typename KT::MemberType; using view_2d = typename KT::template view_2d; - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- struct marineOrganicsTimeState { marineOrganicsTimeState() = default; @@ -50,7 +49,6 @@ struct marineOrganicsFunctions { view_2d emiss_sectors; }; // marineOrganicsData - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- struct marineOrganicsInput { marineOrganicsInput() = default; @@ -69,20 +67,17 @@ struct marineOrganicsFunctions { // signatures using marineOrganicsOutput = marineOrganicsData; - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &marineOrganics_data_file, const std::string &map_file, const std::vector &field_name, const std::string &dim_name1); - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- static std::shared_ptr create_data_reader( const std::shared_ptr &horiz_remapper, const std::string &data_file); - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- static void update_marine_organics_data_from_file( std::shared_ptr &scorpio_reader, @@ -91,7 +86,6 @@ struct marineOrganicsFunctions { AbstractRemapper &horiz_interp, marineOrganicsInput &marineOrganics_input); - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- static void update_marine_organics_timestate( std::shared_ptr &scorpio_reader, @@ -99,21 +93,18 @@ struct marineOrganicsFunctions { marineOrganicsTimeState &time_state, marineOrganicsInput &beg, marineOrganicsInput &end); - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- static void marineOrganics_main(const marineOrganicsTimeState &time_state, const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, const marineOrganicsOutput &data_out); - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- static void perform_time_interpolation( const marineOrganicsTimeState &time_state, const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, const marineOrganicsOutput &data_out); - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- // Performs convex interpolation of x0 and x1 at point t template @@ -121,7 +112,6 @@ struct marineOrganicsFunctions { const ScalarX &x1, const ScalarT &t); - // ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- static void init_marine_organics_file_read( const int &ncol, const std::vector &field_name, diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp index 6eda3a47e72..389445fa024 100644 --- a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp @@ -77,8 +77,6 @@ marineOrganicsFunctions::create_horiz_remapper( } // create_horiz_remapper // ------------------------------------------------------------------------------------------- -// ------------------------------------------------------------------------------------------- - template std::shared_ptr marineOrganicsFunctions::create_data_reader( @@ -92,7 +90,6 @@ marineOrganicsFunctions::create_data_reader( return std::make_shared(data_file, io_grid, io_fields, true); } // create_data_reader -// ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- template void marineOrganicsFunctions::update_marine_organics_data_from_file( @@ -144,7 +141,6 @@ void marineOrganicsFunctions::update_marine_organics_data_from_file( } // END update_marine_organics_data_from_file -// ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- template void marineOrganicsFunctions::update_marine_organics_timestate( @@ -180,7 +176,6 @@ void marineOrganicsFunctions::update_marine_organics_timestate( } // END updata_marine_organics_timestate -// ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- template template @@ -189,7 +184,6 @@ KOKKOS_INLINE_FUNCTION ScalarX marineOrganicsFunctions::linear_interp( return (1 - t) * x0 + t * x1; } // linear_interp -// ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- template void marineOrganicsFunctions::perform_time_interpolation( @@ -239,7 +233,6 @@ void marineOrganicsFunctions::perform_time_interpolation( } // perform_time_interpolation -// ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- template void marineOrganicsFunctions::marineOrganics_main( @@ -270,7 +263,6 @@ void marineOrganicsFunctions::marineOrganics_main( perform_time_interpolation(time_state, data_beg, data_end, data_out); } // marineOrganics_main -// ------------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------------- template void marineOrganicsFunctions::init_marine_organics_file_read( diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp index 223fb78cfc1..8b47c81d907 100644 --- a/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp @@ -14,32 +14,19 @@ struct soilErodibilityFunctions { using KT = KokkosTypes; using const_view_1d = typename KT::template view_1d; - // ------------------------------------------------------------------------------------------- - // ------------------------------------------------------------------------------------------- - - // Soil erodibility routines static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &soilErodibility_data_file, const std::string &map_file, const std::string &field_name, const std::string &dim_name1); - // ------------------------------------------------------------------------------------------- - // ------------------------------------------------------------------------------------------- - static std::shared_ptr create_data_reader( const std::shared_ptr &horiz_remapper, const std::string &data_file); - // ------------------------------------------------------------------------------------------- - // ------------------------------------------------------------------------------------------- - static void update_soil_erodibility_data_from_file( std::shared_ptr &scorpio_reader, AbstractRemapper &horiz_interp, const_view_1d &input); - // ------------------------------------------------------------------------------------------- - // ------------------------------------------------------------------------------------------- - static void init_soil_erodibility_file_read( const int ncol, const std::string field_name, const std::string dim_name1, const std::shared_ptr &grid, diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp index a685134419f..af0c4d73c17 100644 --- a/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp @@ -70,8 +70,6 @@ soilErodibilityFunctions::create_horiz_remapper( } // create_horiz_remapper // ------------------------------------------------------------------------------------------- -// ------------------------------------------------------------------------------------------- - template std::shared_ptr soilErodibilityFunctions::create_data_reader( @@ -86,8 +84,6 @@ soilErodibilityFunctions::create_data_reader( } // create_data_reader // ------------------------------------------------------------------------------------------- -// ------------------------------------------------------------------------------------------- - template void soilErodibilityFunctions::update_soil_erodibility_data_from_file( std::shared_ptr &scorpio_reader, @@ -129,8 +125,6 @@ void soilErodibilityFunctions::update_soil_erodibility_data_from_file( } // END update_soil_erodibility_data_from_file // ------------------------------------------------------------------------------------------- -// ------------------------------------------------------------------------------------------- - template void soilErodibilityFunctions::init_soil_erodibility_file_read( const int ncol, const std::string field_name, const std::string dim_name1, From 8d4ca7677e6cde97fccf948a1685e42744b68188 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 4 Nov 2024 22:33:36 -0800 Subject: [PATCH 352/366] EAMxx: Moves online emiss and constituent init to a new function hpp file --- ...mam_srf_and_online_emissions_functions.hpp | 76 +++++++++++++++++++ ...and_online_emissions_process_interface.cpp | 45 +++++------ ...and_online_emissions_process_interface.hpp | 25 ++---- 3 files changed, 99 insertions(+), 47 deletions(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp new file mode 100644 index 00000000000..c1414d2077f --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp @@ -0,0 +1,76 @@ +#ifndef EAMXX_MAM_SRF_AND_ONLINE_EMISSIONS_FUNCTIONS_HPP +#define EAMXX_MAM_SRF_AND_ONLINE_EMISSIONS_FUNCTIONS_HPP + +namespace scream { + +namespace { + +using KT = ekat::KokkosTypes; +using view_1d = typename KT::template view_1d; +using view_2d = typename KT::template view_2d; +using const_view_1d = typename KT::template view_1d; +using const_view_2d = typename KT::template view_2d; + +//-------- Inititlize gas and aerosol fluxes ------ +void init_fluxes(const int &ncol, + view_2d &constituent_fluxes) { // input-output + + constexpr int pcnst = mam4::aero_model::pcnst; + const int gas_start_ind = mam4::utils::gasses_start_ind(); + + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol, pcnst - gas_start_ind); + + // Parallel loop over all the columns + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const KT::MemberType &team) { + const int icol = team.league_rank(); + view_1d flux_col = ekat::subview(constituent_fluxes, icol); + + // Zero out constituent fluxes only for gasses and aerosols + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, gas_start_ind, pcnst), + [&](int icnst) { flux_col(icnst) = 0; }); + }); + Kokkos::fence(); +} // init_fluxes ends + +//-------- compute online emissions for dust, sea salt and marine organics ----- +void compute_online_dust_nacl_emiss( + const int &ncol, const int &nlev, const const_view_1d &ocnfrac, + const const_view_1d &sst, const const_view_2d &u_wind, + const const_view_2d &v_wind, const const_view_2d &dstflx, + const const_view_1d &mpoly, const const_view_1d &mprot, + const const_view_1d &mlip, const const_view_1d &soil_erodibility, + const const_view_2d &z_mid, + // output + view_2d &constituent_fluxes) { + const int surf_lev = nlev - 1; // surface level + + Kokkos::parallel_for( + "online_emis_fluxes", ncol, KOKKOS_LAMBDA(int icol) { + // Input + const const_view_1d dstflx_icol = ekat::subview(dstflx, icol); + + // Output + view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); + + // Compute online emissions + // NOTE: mam4::aero_model_emissions calculates mass and number emission + // fluxes in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert + mam4::aero_model_emissions::aero_model_emissions( + sst(icol), ocnfrac(icol), u_wind(icol, surf_lev), + v_wind(icol, surf_lev), z_mid(icol, surf_lev), dstflx_icol, + soil_erodibility(icol), mpoly(icol), mprot(icol), mlip(icol), + // out + fluxes_col); + }); + Kokkos::fence(); + +} // compute_online_dust_nacl_emiss ends + +} // namespace +} // namespace scream + +#endif // EAMXX_MAM_SRF_AND_ONLINE_EMISSIONS_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index a3efea7d32d..d410bb01d6e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,5 +1,8 @@ #include +// For surface and online emission functions +#include + // For reading soil erodibility file #include @@ -397,7 +400,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- - preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_, constituent_fluxes_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_); } // end initialize_impl() @@ -412,6 +415,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + // Constituent fluxes [kg/m^2/s] + auto constituent_fluxes = this->constituent_fluxes_; + + // Zero out constituent fluxes only for gasses and aerosols + init_fluxes(ncol_, // in + constituent_fluxes); // in-out + // Gather time and state information for interpolation const auto ts = timestamp() + dt; @@ -419,6 +429,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Online emissions from dust and sea salt //-------------------------------------------------------------------- + // compute_online_dust_nacl_emiss(); + // --- Interpolate marine organics data -- // Update TimeState, note the addition of dt @@ -457,38 +469,17 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Dust fluxes [kg/m^2/s]: Four flux values for each column const const_view_2d dstflx = get_field_in("dstflx").get_view(); - // Constituent fluxes [kg/m^2/s] - auto constituent_fluxes = this->constituent_fluxes_; - // Soil edodibility [fraction] const const_view_1d soil_erodibility = this->soil_erodibility_; // Vertical layer height at midpoints const const_view_2d z_mid = dry_atm_.z_mid; - // TODO: potentially combine with below parfor(icol) loop? - const int surf_lev = nlev_ - 1; // surface level - - Kokkos::parallel_for( - "online_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - // Input - const const_view_1d dstflx_icol = ekat::subview(dstflx, icol); - - // Output - view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); - - // Compute online emissions - // NOTE: mam4::aero_model_emissions calculates mass and number emission - // fluxes in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert - mam4::aero_model_emissions::aero_model_emissions( - sst(icol), ocnfrac(icol), u_wind(icol, surf_lev), - v_wind(icol, surf_lev), z_mid(icol, surf_lev), dstflx_icol, - soil_erodibility(icol), mpoly(icol), mprot(icol), mlip(icol), - // out - fluxes_col); - }); - Kokkos::fence(); - + compute_online_dust_nacl_emiss(ncol_, nlev_, ocnfrac, sst, u_wind, v_wind, + dstflx, mpoly, mprot, mlip, soil_erodibility, + z_mid, + // output + constituent_fluxes); //-------------------------------------------------------------------- // Interpolate srf emiss data read in from emissions files //-------------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index ae00098b079..1a3bb4f36e3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -103,13 +103,11 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // on host: initializes preprocess functor with necessary state data void initialize(const int &ncol, const int &nlev, const mam_coupling::WetAtmosphere &wet_atm, - const mam_coupling::DryAtmosphere &dry_atm, - const view_2d &constituent_fluxes) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; - dry_atm_pre_ = dry_atm; - constituent_fluxes_pre_ = constituent_fluxes; + const mam_coupling::DryAtmosphere &dry_atm) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + dry_atm_pre_ = dry_atm; } KOKKOS_INLINE_FUNCTION void operator()( @@ -122,18 +120,6 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // for atmosphere compute_vertical_layer_heights(team, dry_atm_pre_, icol); compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, icol); - - view_1d flux_col = ekat::subview(constituent_fluxes_pre_, icol); - - // Zero out constituent fluxes only for gasses and aerosols - const int pcnst = mam4::aero_model::pcnst; - const int gas_start_ind = mam4::utils::gasses_start_ind(); - - // FIXME: Is there a better way to zero out a select indices? - // We should probably do it in run_impl directly - for(int ispc = gas_start_ind; ispc < pcnst; ++ispc) { - flux_col(ispc) = 0; - } } // Preprocess operator() // local variables for preprocess struct @@ -143,7 +129,6 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // local atmospheric and aerosol state data mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; - view_2d constituent_fluxes_pre_; }; // MAMSrfOnlineEmiss::Preprocess private: // preprocessing scratch pad From a20712c19505fbcf1025d5d6ce1d29952c25a772 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Tue, 5 Nov 2024 11:57:27 -0700 Subject: [PATCH 353/366] fix cuda compile warnings --- ...amxx_mam_srf_and_online_emissions_process_interface.cpp | 7 ++++--- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index d410bb01d6e..58a1c341fa5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -509,15 +509,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { auto fluxes_in_mks_units = this->fluxes_in_mks_units_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; + const view_1d ispec_outdata0 = + ekat::subview(ispec_srf.data_out_.emiss_sectors, 0); // Parallel loop over all the columns to update units Kokkos::parallel_for( "srf_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - fluxes_in_mks_units(icol) = - ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; + fluxes_in_mks_units(icol) = ispec_outdata0(icol) * mfactor; constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); }); } // for loop for species Kokkos::fence(); } // run_impl ends // ============================================================================= -} // namespace scream \ No newline at end of file +} // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index fa037dc281d..b8ebfdbe501 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -179,7 +179,7 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int time_index, // zero-based AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input) { using namespace ShortFieldTagsNames; - + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); // 1. Read from file From a40341e657253fd3b810a3c2eb3b6f3682e2f249 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 13 Nov 2024 12:54:49 -0800 Subject: [PATCH 354/366] EAMxx:Inits constituent fluxes to zero --- components/eamxx/cime_config/namelist_defaults_scream.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 9c11c76aea1..d23169173ce 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -638,6 +638,7 @@ be lost if SCREAM_HACK_XML is not enabled. 1.37146e-07 ,3.45899e-08 ,1.00000e-06 ,9.99601e-08 1.37452e-07 ,3.46684e-08 ,1.00900e-06 ,9.99601e-08 5.08262e-12 ,1.54035e-13 ,3.09018e-13 ,9.14710e-22 + 0.0 0.0 0.0 0.0 From df4f13c1942429b87f049e5326ac5b01d24ec58f Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Fri, 15 Nov 2024 13:01:30 -0700 Subject: [PATCH 355/366] Revert "EAMxx: Clang format" This reverts commit 22086f144e9a8ef9264c383f5487b39d8de37319. --- ...mxx_mam_microphysics_process_interface.cpp | 109 +++++++----------- 1 file changed, 44 insertions(+), 65 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 328afc89690..35f27e1db00 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -2,11 +2,7 @@ // impl namespace for some driver level functions for microphysics -<<<<<<< HEAD #include "readfiles/photo_table_utils.cpp" -#include "readfiles/find_season_index_utils.hpp" -======= ->>>>>>> EAMxx: Clang format #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" #include "readfiles/photo_table_utils.cpp" @@ -211,7 +207,7 @@ void MAMMicrophysics::set_grids( LinozHorizInterp_, linoz_file_name_); // linoz reader - const auto io_grid_linoz = LinozHorizInterp_->get_tgt_grid(); + const auto io_grid_linoz = LinozHorizInterp_->get_src_grid(); const int num_cols_io_linoz = io_grid_linoz->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_linoz = @@ -239,7 +235,7 @@ void MAMMicrophysics::set_grids( TracerHorizInterp_, oxid_file_name_); const int nvars = int(var_names.size()); - const auto io_grid = TracerHorizInterp_->get_tgt_grid(); + const auto io_grid = TracerHorizInterp_->get_src_grid(); const int num_cols_io = io_grid->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io = @@ -264,88 +260,78 @@ void MAMMicrophysics::set_grids( "num_a1", "num_a2", "num_a4", "soag"}; for(const auto &var_name : extfrc_lst_) { - std::string item_name = "mam4_" + var_name + "_elevated_emiss_file_name"; + std::string item_name = "mam4_" + var_name + "_verti_emiss_file_name"; const auto file_name = m_params.get(item_name); - elevated_emis_file_name_[var_name] = file_name; + vert_emis_file_name_[var_name] = file_name; } - elevated_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; - elevated_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; - elevated_emis_var_names_["so4_a2"] = {"contvolc"}; - elevated_emis_var_names_["pom_a4"] = {"BB"}; - elevated_emis_var_names_["bc_a4"] = {"BB"}; - elevated_emis_var_names_["num_a1"] = { + vert_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; + vert_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; + vert_emis_var_names_["so4_a2"] = {"contvolc"}; + vert_emis_var_names_["pom_a4"] = {"BB"}; + vert_emis_var_names_["bc_a4"] = {"BB"}; + vert_emis_var_names_["num_a1"] = { "num_a1_SO4_ELEV_BB", "num_a1_SO4_ELEV_ENE", "num_a1_SO4_ELEV_IND", "num_a1_SO4_ELEV_contvolc"}; - elevated_emis_var_names_["num_a2"] = {"num_a2_SO4_ELEV_contvolc"}; + vert_emis_var_names_["num_a2"] = {"num_a2_SO4_ELEV_contvolc"}; // num_a4 // FIXME: why the sectors in this files are num_a1; // I guess this should be num_a4? Is this a bug in the orginal nc files? - elevated_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", + vert_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", "num_a1_POM_ELEV_BB"}; - elevated_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; + vert_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; - int elevated_emiss_cyclical_ymd = m_params.get("elevated_emiss_ymd"); + int verti_emiss_cyclical_ymd = m_params.get("verti_emiss_ymd"); for(const auto &var_name : extfrc_lst_) { - const auto file_name = elevated_emis_file_name_[var_name]; - const auto var_names = elevated_emis_var_names_[var_name]; + const auto file_name = vert_emis_file_name_[var_name]; + const auto var_names = vert_emis_var_names_[var_name]; scream::mam_coupling::TracerData data_tracer; scream::mam_coupling::setup_tracer_data(data_tracer, file_name, - elevated_emiss_cyclical_ymd); + verti_emiss_cyclical_ymd); auto hor_rem = scream::mam_coupling::create_horiz_remapper( grid_, file_name, extfrc_map_file, var_names, data_tracer); - auto file_reader = - scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name, - data_tracer.file_type); - ElevatedEmissionsHorizInterp_.push_back(hor_rem); - ElevatedEmissionsDataReader_.push_back(file_reader); - elevated_emis_data_.push_back(data_tracer); - } // var_name elevated emissions + scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name); + VertEmissionsHorizInterp_.push_back(hor_rem); + VertEmissionsDataReader_.push_back(file_reader); + vert_emis_data_.push_back(data_tracer); + } // var_name vert emissions int i = 0; int offset_emis_ver = 0; for(const auto &var_name : extfrc_lst_) { - const auto file_name = elevated_emis_file_name_[var_name]; - const auto var_names = elevated_emis_var_names_[var_name]; + const auto file_name = vert_emis_file_name_[var_name]; + const auto var_names = vert_emis_var_names_[var_name]; const int nvars = static_cast(var_names.size()); forcings_[i].nsectors = nvars; // I am assuming the order of species in extfrc_lst_. // Indexing in mam4xx is fortran. forcings_[i].frc_ndx = i + 1; - const auto io_grid_emis = ElevatedEmissionsHorizInterp_[i]->get_tgt_grid(); + const auto io_grid_emis = VertEmissionsHorizInterp_[i]->get_src_grid(); const int num_cols_io_emis = io_grid_emis->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_emis = io_grid_emis ->get_num_vertical_levels(); // Number of levels per column - elevated_emis_data_[i].init(num_cols_io_emis, num_levs_io_emis, nvars); - elevated_emis_data_[i].allocate_temporal_views(); - forcings_[i].file_alt_data = elevated_emis_data_[i].has_altitude_; + vert_emis_data_[i].init(num_cols_io_emis, num_levs_io_emis, nvars); + vert_emis_data_[i].allocate_temporal_views(); + forcings_[i].file_alt_data = vert_emis_data_[i].has_altitude_; for(int isp = 0; isp < nvars; ++isp) { forcings_[i].offset = offset_emis_ver; - elevated_emis_output_[isp + offset_emis_ver] = - view_2d("elevated_emis_output_", ncol_, nlev_); + vert_emis_output_[isp + offset_emis_ver] = + view_2d("vert_emis_output_", ncol_, nlev_); } offset_emis_ver += nvars; ++i; } // end i EKAT_REQUIRE_MSG( - offset_emis_ver <= int(mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS), + offset_emis_ver <= int(mam_coupling::MAX_NUM_VERT_EMISSION_FIELDS), "Error! Number of fields is bigger than " - "MAX_NUM_ELEVATED_EMISSIONS_FIELDS. Increase the " - "MAX_NUM_ELEVATED_EMISSIONS_FIELDS in tracer_reader_utils.hpp \n"); + "MAX_NUM_VERT_EMISSION_FIELDS. Increase the " + "MAX_NUM_VERT_EMISSION_FIELDS in tracer_reader_utils.hpp \n"); } // Tracer external forcing data - - { - const std::string season_wes_file = m_params.get("mam4_season_wes_file"); - const auto& clat = col_latitudes_; - mam_coupling::find_season_index_reader(season_wes_file, - clat, - index_season_lai_); - } } // set_grids // ================================================================ @@ -534,8 +520,8 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { for(int i = 0; i < static_cast(extfrc_lst_.size()); ++i) { scream::mam_coupling::update_tracer_data_from_file( - ElevatedEmissionsDataReader_[i], curr_month, *ElevatedEmissionsHorizInterp_[i], - elevated_emis_data_[i]); + VertEmissionsDataReader_[i], curr_month, *VertEmissionsHorizInterp_[i], + vert_emis_data_[i]); } invariants_ = view_3d("invarians", ncol_, nlev_, mam4::gas_chemistry::nfs); @@ -632,20 +618,20 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_output); // out Kokkos::fence(); - elevated_emiss_time_state_.t_now = ts.frac_of_year_in_days(); + vert_emiss_time_state_.t_now = ts.frac_of_year_in_days(); int i = 0; for(const auto &var_name : extfrc_lst_) { - const auto file_name = elevated_emis_file_name_[var_name]; - const auto var_names = elevated_emis_var_names_[var_name]; + const auto file_name = vert_emis_file_name_[var_name]; + const auto var_names = vert_emis_var_names_[var_name]; const int nsectors = int(var_names.size()); - view_2d elevated_emis_output[nsectors]; + view_2d vert_emis_output[nsectors]; for(int isp = 0; isp < nsectors; ++isp) { - elevated_emis_output[isp] = elevated_emis_output_[isp + forcings_[i].offset]; + vert_emis_output[isp] = vert_emis_output_[isp + forcings_[i].offset]; } scream::mam_coupling::advance_tracer_data( - ElevatedEmissionsDataReader_[i], *ElevatedEmissionsHorizInterp_[i], ts, - elevated_emiss_time_state_, elevated_emis_data_[i], dry_atm_.p_mid, - dry_atm_.z_iface, elevated_emis_output); + VertEmissionsDataReader_[i], *VertEmissionsHorizInterp_[i], ts, + vert_emiss_time_state_, vert_emis_data_[i], dry_atm_.p_mid, + dry_atm_.z_iface, vert_emis_output); i++; Kokkos::fence(); } @@ -720,17 +706,10 @@ void MAMMicrophysics::run_impl(const double dt) { const auto zenith_angle = acos_cosine_zenith_; constexpr int gas_pcnst = mam_coupling::gas_pcnst(); -<<<<<<< HEAD - const auto& elevated_emis_output = elevated_emis_output_; - const auto& extfrc = extfrc_; - const auto& forcings = forcings_; - constexpr int extcnt = mam4::gas_chemistry::extcnt; -======= const auto &vert_emis_output = vert_emis_output_; const auto &extfrc = extfrc_; const auto &forcings = forcings_; constexpr int extcnt = mam4::gas_chemistry::extcnt; ->>>>>>> EAMxx: Clang format const int offset_aerosol = mam4::utils::gasses_start_ind(); Real adv_mass_kg_per_moles[gas_pcnst]; @@ -779,7 +758,7 @@ void MAMMicrophysics::run_impl(const double dt) { // We may need to move this line where we read files. forcings_in[i].file_alt_data = file_alt_data; for(int isec = 0; isec < forcings[i].nsectors; ++isec) { - const auto field = elevated_emis_output[isec + forcings[i].offset]; + const auto field = vert_emis_output[isec + forcings[i].offset]; forcings_in[i].fields_data[isec] = ekat::subview(field, icol); } } // extcnt for loop From 60a90e92b05069ab136fccbf464486eb7976b3e5 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Fri, 15 Nov 2024 15:44:22 -0700 Subject: [PATCH 356/366] ff mam4xx to current main --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index d1c70ab972b..e26b5b6faa5 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit d1c70ab972bc00269939fd3296d93a719f273105 +Subproject commit e26b5b6faa5cdf39a5421328165c49bdb728b038 From c8c4d6592bd32f88fd3c6bb23a41ebea566b591b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 15 Nov 2024 15:54:39 -0800 Subject: [PATCH 357/366] EAMxx: Remove all diffs from microphysics cpp file --- ...mxx_mam_microphysics_process_interface.cpp | 131 ++++++++++-------- 1 file changed, 70 insertions(+), 61 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 35f27e1db00..57eaf927548 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -3,8 +3,8 @@ // impl namespace for some driver level functions for microphysics #include "readfiles/photo_table_utils.cpp" +#include "readfiles/find_season_index_utils.hpp" #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" -#include "readfiles/photo_table_utils.cpp" namespace scream { @@ -36,7 +36,7 @@ MAMMicrophysics::MAMMicrophysics(const ekat::Comm &comm, config_.linoz.o3_lbl = m_params.get("mam4_o3_lbl"); config_.linoz.o3_tau = m_params.get("mam4_o3_tau"); config_.linoz.o3_sfc = m_params.get("mam4_o3_sfc"); - config_.linoz.psc_T = m_params.get("mam4_psc_T"); + config_.linoz.psc_T = m_params.get("mam4_psc_T"); } AtmosphereProcessType MAMMicrophysics::type() const { @@ -81,20 +81,19 @@ void MAMMicrophysics::set_grids( // ----------- Atmospheric quantities ------------- // Specific humidity [kg/kg](Require only for building DS) - add_tracer("qv", grid_, kg / kg); // specific humidity + add_tracer("qv", grid_, kg/kg); // specific humidity // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) - add_tracer("qc", grid_, kg / kg); // cloud liquid wet mixing ratio + add_tracer("qc", grid_, kg/kg); // cloud liquid wet mixing ratio // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) - add_tracer("qi", grid_, kg / kg); // ice wet mixing ratio + add_tracer("qi", grid_, kg/kg); // ice wet mixing ratio // Cloud liquid number mixing ratio [1/kg](Require only for building DS) - add_tracer("nc", grid_, - n_unit); // cloud liquid wet number mixing ratio + add_tracer("nc", grid_, n_unit); // cloud liquid wet number mixing ratio // Cloud ice number mixing ratio [1/kg](Require only for building DS) - add_tracer("ni", grid_, n_unit); // ice number mixing ratio + add_tracer("ni", grid_, n_unit); // ice number mixing ratio // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); @@ -162,7 +161,7 @@ void MAMMicrophysics::set_grids( mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - add_tracer(int_mmr_field_name, grid_, kg / kg); + add_tracer(int_mmr_field_name, grid_, kg/kg); } } // for loop species } // for loop nmodes interstitial @@ -184,7 +183,7 @@ void MAMMicrophysics::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_tracer(gas_mmr_field_name, grid_, kg / kg); + add_tracer(gas_mmr_field_name, grid_, kg/kg); } // Creating a Linoz reader and setting Linoz parameters involves reading data @@ -207,7 +206,7 @@ void MAMMicrophysics::set_grids( LinozHorizInterp_, linoz_file_name_); // linoz reader - const auto io_grid_linoz = LinozHorizInterp_->get_src_grid(); + const auto io_grid_linoz = LinozHorizInterp_->get_tgt_grid(); const int num_cols_io_linoz = io_grid_linoz->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_linoz = @@ -235,7 +234,7 @@ void MAMMicrophysics::set_grids( TracerHorizInterp_, oxid_file_name_); const int nvars = int(var_names.size()); - const auto io_grid = TracerHorizInterp_->get_src_grid(); + const auto io_grid = TracerHorizInterp_->get_tgt_grid(); const int num_cols_io = io_grid->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io = @@ -260,78 +259,88 @@ void MAMMicrophysics::set_grids( "num_a1", "num_a2", "num_a4", "soag"}; for(const auto &var_name : extfrc_lst_) { - std::string item_name = "mam4_" + var_name + "_verti_emiss_file_name"; + std::string item_name = "mam4_" + var_name + "_elevated_emiss_file_name"; const auto file_name = m_params.get(item_name); - vert_emis_file_name_[var_name] = file_name; + elevated_emis_file_name_[var_name] = file_name; } - vert_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; - vert_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; - vert_emis_var_names_["so4_a2"] = {"contvolc"}; - vert_emis_var_names_["pom_a4"] = {"BB"}; - vert_emis_var_names_["bc_a4"] = {"BB"}; - vert_emis_var_names_["num_a1"] = { + elevated_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; + elevated_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; + elevated_emis_var_names_["so4_a2"] = {"contvolc"}; + elevated_emis_var_names_["pom_a4"] = {"BB"}; + elevated_emis_var_names_["bc_a4"] = {"BB"}; + elevated_emis_var_names_["num_a1"] = { "num_a1_SO4_ELEV_BB", "num_a1_SO4_ELEV_ENE", "num_a1_SO4_ELEV_IND", "num_a1_SO4_ELEV_contvolc"}; - vert_emis_var_names_["num_a2"] = {"num_a2_SO4_ELEV_contvolc"}; + elevated_emis_var_names_["num_a2"] = {"num_a2_SO4_ELEV_contvolc"}; // num_a4 // FIXME: why the sectors in this files are num_a1; // I guess this should be num_a4? Is this a bug in the orginal nc files? - vert_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", + elevated_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", "num_a1_POM_ELEV_BB"}; - vert_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; + elevated_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; - int verti_emiss_cyclical_ymd = m_params.get("verti_emiss_ymd"); + int elevated_emiss_cyclical_ymd = m_params.get("elevated_emiss_ymd"); for(const auto &var_name : extfrc_lst_) { - const auto file_name = vert_emis_file_name_[var_name]; - const auto var_names = vert_emis_var_names_[var_name]; + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; scream::mam_coupling::TracerData data_tracer; scream::mam_coupling::setup_tracer_data(data_tracer, file_name, - verti_emiss_cyclical_ymd); + elevated_emiss_cyclical_ymd); auto hor_rem = scream::mam_coupling::create_horiz_remapper( grid_, file_name, extfrc_map_file, var_names, data_tracer); + auto file_reader = - scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name); - VertEmissionsHorizInterp_.push_back(hor_rem); - VertEmissionsDataReader_.push_back(file_reader); - vert_emis_data_.push_back(data_tracer); - } // var_name vert emissions + scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name, + data_tracer.file_type); + ElevatedEmissionsHorizInterp_.push_back(hor_rem); + ElevatedEmissionsDataReader_.push_back(file_reader); + elevated_emis_data_.push_back(data_tracer); + } // var_name elevated emissions int i = 0; int offset_emis_ver = 0; for(const auto &var_name : extfrc_lst_) { - const auto file_name = vert_emis_file_name_[var_name]; - const auto var_names = vert_emis_var_names_[var_name]; + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; const int nvars = static_cast(var_names.size()); forcings_[i].nsectors = nvars; // I am assuming the order of species in extfrc_lst_. // Indexing in mam4xx is fortran. forcings_[i].frc_ndx = i + 1; - const auto io_grid_emis = VertEmissionsHorizInterp_[i]->get_src_grid(); + const auto io_grid_emis = ElevatedEmissionsHorizInterp_[i]->get_tgt_grid(); const int num_cols_io_emis = io_grid_emis->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_emis = io_grid_emis ->get_num_vertical_levels(); // Number of levels per column - vert_emis_data_[i].init(num_cols_io_emis, num_levs_io_emis, nvars); - vert_emis_data_[i].allocate_temporal_views(); - forcings_[i].file_alt_data = vert_emis_data_[i].has_altitude_; + elevated_emis_data_[i].init(num_cols_io_emis, num_levs_io_emis, nvars); + elevated_emis_data_[i].allocate_temporal_views(); + forcings_[i].file_alt_data = elevated_emis_data_[i].has_altitude_; for(int isp = 0; isp < nvars; ++isp) { forcings_[i].offset = offset_emis_ver; - vert_emis_output_[isp + offset_emis_ver] = - view_2d("vert_emis_output_", ncol_, nlev_); + elevated_emis_output_[isp + offset_emis_ver] = + view_2d("elevated_emis_output_", ncol_, nlev_); } offset_emis_ver += nvars; ++i; } // end i EKAT_REQUIRE_MSG( - offset_emis_ver <= int(mam_coupling::MAX_NUM_VERT_EMISSION_FIELDS), + offset_emis_ver <= int(mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS), "Error! Number of fields is bigger than " - "MAX_NUM_VERT_EMISSION_FIELDS. Increase the " - "MAX_NUM_VERT_EMISSION_FIELDS in tracer_reader_utils.hpp \n"); + "MAX_NUM_ELEVATED_EMISSIONS_FIELDS. Increase the " + "MAX_NUM_ELEVATED_EMISSIONS_FIELDS in tracer_reader_utils.hpp \n"); } // Tracer external forcing data + + { + const std::string season_wes_file = m_params.get("mam4_season_wes_file"); + const auto& clat = col_latitudes_; + mam_coupling::find_season_index_reader(season_wes_file, + clat, + index_season_lai_); + } } // set_grids // ================================================================ @@ -520,8 +529,8 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { for(int i = 0; i < static_cast(extfrc_lst_.size()); ++i) { scream::mam_coupling::update_tracer_data_from_file( - VertEmissionsDataReader_[i], curr_month, *VertEmissionsHorizInterp_[i], - vert_emis_data_[i]); + ElevatedEmissionsDataReader_[i], curr_month, *ElevatedEmissionsHorizInterp_[i], + elevated_emis_data_[i]); } invariants_ = view_3d("invarians", ncol_, nlev_, mam4::gas_chemistry::nfs); @@ -618,20 +627,20 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_output); // out Kokkos::fence(); - vert_emiss_time_state_.t_now = ts.frac_of_year_in_days(); + elevated_emiss_time_state_.t_now = ts.frac_of_year_in_days(); int i = 0; for(const auto &var_name : extfrc_lst_) { - const auto file_name = vert_emis_file_name_[var_name]; - const auto var_names = vert_emis_var_names_[var_name]; + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; const int nsectors = int(var_names.size()); - view_2d vert_emis_output[nsectors]; + view_2d elevated_emis_output[nsectors]; for(int isp = 0; isp < nsectors; ++isp) { - vert_emis_output[isp] = vert_emis_output_[isp + forcings_[i].offset]; + elevated_emis_output[isp] = elevated_emis_output_[isp + forcings_[i].offset]; } scream::mam_coupling::advance_tracer_data( - VertEmissionsDataReader_[i], *VertEmissionsHorizInterp_[i], ts, - vert_emiss_time_state_, vert_emis_data_[i], dry_atm_.p_mid, - dry_atm_.z_iface, vert_emis_output); + ElevatedEmissionsDataReader_[i], *ElevatedEmissionsHorizInterp_[i], ts, + elevated_emiss_time_state_, elevated_emis_data_[i], dry_atm_.p_mid, + dry_atm_.z_iface, elevated_emis_output); i++; Kokkos::fence(); } @@ -706,10 +715,10 @@ void MAMMicrophysics::run_impl(const double dt) { const auto zenith_angle = acos_cosine_zenith_; constexpr int gas_pcnst = mam_coupling::gas_pcnst(); - const auto &vert_emis_output = vert_emis_output_; - const auto &extfrc = extfrc_; - const auto &forcings = forcings_; - constexpr int extcnt = mam4::gas_chemistry::extcnt; + const auto& elevated_emis_output = elevated_emis_output_; + const auto& extfrc = extfrc_; + const auto& forcings = forcings_; + constexpr int extcnt = mam4::gas_chemistry::extcnt; const int offset_aerosol = mam4::utils::gasses_start_ind(); Real adv_mass_kg_per_moles[gas_pcnst]; @@ -758,7 +767,7 @@ void MAMMicrophysics::run_impl(const double dt) { // We may need to move this line where we read files. forcings_in[i].file_alt_data = file_alt_data; for(int isec = 0; isec < forcings[i].nsectors; ++isec) { - const auto field = vert_emis_output[isec + forcings[i].offset]; + const auto field = elevated_emis_output[isec + forcings[i].offset]; forcings_in[i].fields_data[isec] = ekat::subview(field, icol); } } // extcnt for loop @@ -800,9 +809,9 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_o3col_clim_icol, linoz_PmL_clim_icol, linoz_dPmL_dO3_icol, linoz_dPmL_dT_icol, linoz_dPmL_dO3col_icol, linoz_cariolle_pscs_icol, eccf, adv_mass_kg_per_moles, clsmap_4, - permute_4, offset_aerosol, config.linoz.o3_sfc, config.linoz.o3_tau, - config.linoz.o3_lbl, dry_diameter_icol, wet_diameter_icol, - wetdens_icol); + permute_4, offset_aerosol, + config.linoz.o3_sfc, config.linoz.o3_tau, config.linoz.o3_lbl, + dry_diameter_icol, wet_diameter_icol, wetdens_icol); }); // parallel_for for the column loop Kokkos::fence(); From e02b76cdf93a8c1ce09e06b9fbfa26de3856ae73 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 15 Nov 2024 20:18:34 -0800 Subject: [PATCH 358/366] EAMxx: Adds missing namelist entries for multi process test --- .../input.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index 55c62d69aa2..48045295050 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -23,7 +23,9 @@ atmosphere_processes: srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc - + + soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc + marine_organics_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -40,6 +42,13 @@ initial_conditions: topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} pbl_height: 0.0 + dstflx : 0.0 + ocnfrac: 0.10000000000000000E+001 + sst: 0.30178553874977507E+003 + cldfrac_tot: 0.138584624960092 + horiz_winds: [-0.24988988196194634E+000, -0.23959782871450760E+000] + constituent_fluxes: 0.0 + # The parameters for I/O control Scorpio: From 71e4bc5059aebad00c8c7ae82309422b4fc1cb06 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 16 Nov 2024 10:59:15 -0800 Subject: [PATCH 359/366] EAMxx: Moves fences into the interface and remove extra comments --- components/eamxx/cime_config/namelist_defaults_scream.xml | 5 ----- .../mam/eamxx_mam_srf_and_online_emissions_functions.hpp | 3 --- .../eamxx_mam_srf_and_online_emissions_process_interface.cpp | 5 ++--- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index d23169173ce..771928a8122 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -372,15 +372,10 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/dst_ne30pg2_c20241028.nc - - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/dst_ne4pg2_c20241028.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne30pg2_c20241030.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne4pg2_c20241030.nc diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp index c1414d2077f..9c01daf8223 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp @@ -33,7 +33,6 @@ void init_fluxes(const int &ncol, Kokkos::TeamVectorRange(team, gas_start_ind, pcnst), [&](int icnst) { flux_col(icnst) = 0; }); }); - Kokkos::fence(); } // init_fluxes ends //-------- compute online emissions for dust, sea salt and marine organics ----- @@ -66,8 +65,6 @@ void compute_online_dust_nacl_emiss( // out fluxes_col); }); - Kokkos::fence(); - } // compute_online_dust_nacl_emiss ends } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 58a1c341fa5..fd3ebf79700 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -421,7 +421,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Zero out constituent fluxes only for gasses and aerosols init_fluxes(ncol_, // in constituent_fluxes); // in-out - + Kokkos::fence(); // Gather time and state information for interpolation const auto ts = timestamp() + dt; @@ -429,8 +429,6 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Online emissions from dust and sea salt //-------------------------------------------------------------------- - // compute_online_dust_nacl_emiss(); - // --- Interpolate marine organics data -- // Update TimeState, note the addition of dt @@ -480,6 +478,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { z_mid, // output constituent_fluxes); + Kokkos::fence(); //-------------------------------------------------------------------- // Interpolate srf emiss data read in from emissions files //-------------------------------------------------------------------- From b9333f513e64a92ed275d022f8ae650656b53b86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:51:43 +0000 Subject: [PATCH 360/366] Bump DavidAnson/markdownlint-cli2-action from 17 to 18 Bumps [DavidAnson/markdownlint-cli2-action](https://github.com/davidanson/markdownlint-cli2-action) from 17 to 18. - [Release notes](https://github.com/davidanson/markdownlint-cli2-action/releases) - [Commits](https://github.com/davidanson/markdownlint-cli2-action/compare/v17...v18) --- updated-dependencies: - dependency-name: DavidAnson/markdownlint-cli2-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-md-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 46319b08658..ad24487695e 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -27,7 +27,7 @@ jobs: with: files: '**/*.md' separator: "," - - uses: DavidAnson/markdownlint-cli2-action@v17 + - uses: DavidAnson/markdownlint-cli2-action@v18 if: steps.changed-files.outputs.any_changed == 'true' with: config: 'docs/.markdownlint.json' From d0f2a5058196c6b984f89c537709f1794d7e37da Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 18 Nov 2024 17:33:01 -0700 Subject: [PATCH 361/366] EAMxx: Modify microphysics interface to include sethet (washout rates) computations. --- ...mxx_mam_microphysics_process_interface.cpp | 32 ++++++++++++++++++- ...mxx_mam_microphysics_process_interface.hpp | 3 ++ .../mam/aero_microphys/input.yaml | 3 +- externals/mam4xx | 2 +- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 57eaf927548..4a674fb094d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -146,6 +146,15 @@ void MAMMicrophysics::set_grids( // surface albedo shortwave, direct add_field("sfc_alb_dir_vis", scalar2d, nondim, grid_name); + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] + add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); + + // Stratiform rain production rate [kg/kg/s] + add_field("precip_total_tend", scalar3d_mid, kg / kg / s, + grid_name); + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- @@ -512,6 +521,9 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { const int photo_table_len = get_photo_table_work_len(photo_table_); work_photo_table_ = view_2d("work_photo_table", ncol_, photo_table_len); + const int sethet_work_len = mam4::mo_sethet::get_total_work_len_sethet(); + work_set_het_ = view_2d("work_set_het_array", ncol_, sethet_work_len); + cmfdqr_ = view_1d("cmfdqr_", nlev_); // here's where we store per-column photolysis rates photo_rates_ = view_3d("photo_rates", ncol_, nlev_, mam4::mo_photo::phtcnt); @@ -565,6 +577,14 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] + const auto& nevapr = get_field_in("nevapr").get_view(); + + // Stratiform rain production rate [kg/kg/s] + const auto& prain = get_field_in("precip_total_tend").get_view(); + const auto wet_geometric_mean_diameter_i = get_field_in("dgnumwet").get_view(); const auto dry_geometric_mean_diameter_i = @@ -733,6 +753,8 @@ void MAMMicrophysics::run_impl(const double dt) { clsmap_4[i] = mam4::gas_chemistry::clsmap_4[i]; permute_4[i] = mam4::gas_chemistry::permute_4[i]; } + const auto& cmfdqr = cmfdqr_; + const auto& work_set_het =work_set_het_; // loop over atmosphere columns and compute aerosol microphyscs Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { @@ -798,6 +820,9 @@ void MAMMicrophysics::run_impl(const double dt) { ekat::subview(linoz_dPmL_dO3col, icol); const auto linoz_cariolle_pscs_icol = ekat::subview(linoz_cariolle_pscs, icol); + const auto nevapr_icol = ekat::subview(nevapr, icol); + const auto prain_icol = ekat::subview(prain, icol); + const auto work_set_het_icol = ekat::subview(work_set_het, icol); // Note: All variables are inputs, except for progs, which is an // input/output variable. mam4::microphysics::perform_atmospheric_chemistry_and_microphysics( @@ -811,7 +836,12 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_cariolle_pscs_icol, eccf, adv_mass_kg_per_moles, clsmap_4, permute_4, offset_aerosol, config.linoz.o3_sfc, config.linoz.o3_tau, config.linoz.o3_lbl, - dry_diameter_icol, wet_diameter_icol, wetdens_icol); + dry_diameter_icol, wet_diameter_icol, wetdens_icol, + dry_atm.phis(icol), + cmfdqr, + prain_icol, + nevapr_icol, + work_set_het_icol); }); // parallel_for for the column loop Kokkos::fence(); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp index d555fe82024..6ff846d0d0c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -242,6 +242,9 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { view_1d acos_cosine_zenith_; view_int_2d index_season_lai_; + // // dq/dt for convection [kg/kg/s] + view_1d cmfdqr_; + view_2d work_set_het_; }; // MAMMicrophysics diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml index 9348a04a21d..7dd28a87f47 100644 --- a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml +++ b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml @@ -58,7 +58,8 @@ initial_conditions: dgnum: [1.246662106183775E-007, 4.081134799487888E-008, 1.103139143795796E-006, 1.000000011686097E-007] dgnumwet: [2.367209731605067E-007, 6.780643470563889E-008, 3.028011448344027E-006, 1.000000096285154E-007] wetdens: [1038.67760516297, 1046.20002003441, 1031.74623165457, 1086.79731859184] - + nevapr: 0.0 + precip_total_tend: 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] diff --git a/externals/mam4xx b/externals/mam4xx index e26b5b6faa5..7d7a4a673a1 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit e26b5b6faa5cdf39a5421328165c49bdb728b038 +Subproject commit 7d7a4a673a1f2709986b62c6490c8414829c5212 From 00d55a053819a5ce3b2e13d534f004f9b07fb49a Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 19 Nov 2024 11:07:44 -0700 Subject: [PATCH 362/366] EAMxx: Update mam4xx submodule to fix CUDA test. --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 7d7a4a673a1..fdbb0816c5c 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 7d7a4a673a1f2709986b62c6490c8414829c5212 +Subproject commit fdbb0816c5c0c541265ec17f544908da935d8af6 From 2b50dbbed5730efe7819fa2845b35afd841ad464 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Fri, 8 Nov 2024 14:21:04 -0800 Subject: [PATCH 363/366] Add standard name metadata to most EAMxx output variables This commit brings EAMxx into CF-Compliance by adding `standard_name` metadata for a subset of the EAMxx output variables that have one. --- .../eamxx/src/share/io/scorpio_output.cpp | 8 +- .../eamxx/src/share/io/scorpio_output.hpp | 2 +- .../eamxx/src/share/io/scream_io_utils.hpp | 84 ++++++++++++++++++- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 87fe9af7133..e8f322b85f3 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1044,9 +1044,15 @@ register_variables(const std::string& filename, // Gather longname (if not already in the io: string attributes) if (str_atts.count("long_name")==0) { - auto longname = m_longnames.get_longname(name); + auto longname = m_default_metadata.get_longname(name); scorpio::set_attribute(filename, name, "long_name", longname); } + + // Gather standard name, CF-Compliant (if not already in the io: string attributes) + if (str_atts.count("standard_name")==0) { + auto standardname = m_default_metadata.get_standardname(name); + scorpio::set_attribute(filename, name, "standard_name", standardname); + } } } // Now register the average count variables diff --git a/components/eamxx/src/share/io/scorpio_output.hpp b/components/eamxx/src/share/io/scorpio_output.hpp index 2cb45f28a32..696e08c9982 100644 --- a/components/eamxx/src/share/io/scorpio_output.hpp +++ b/components/eamxx/src/share/io/scorpio_output.hpp @@ -206,7 +206,7 @@ class AtmosphereOutput std::map> m_diagnostics; std::map> m_diag_depends_on_diags; std::map m_diag_computed; - LongNames m_longnames; + DefaultMetadata m_default_metadata; // Use float, so that if output fp_precision=float, this is a representable value. // Otherwise, you would get an error from Netcdf, like diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index d6c36ab3758..796ecd2a003 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -72,7 +72,7 @@ std::string find_filename_in_rpointer ( const OutputAvgType avg_type = OutputAvgType::Instant, const IOControl& control = {}); -struct LongNames { +struct DefaultMetadata { std::string get_longname (const std::string& name) { if (name_2_longname.count(name)>0) { @@ -83,6 +83,15 @@ struct LongNames { } } + std::string get_standardname (const std::string& name) { + if (name_2_standardname.count(name)>0) { + return name_2_standardname.at(name); + } else { + // TODO: Do we want to print a Warning message? I'm not sure if its needed. + return name; + } + } + // Create map of longnames, can be added to as developers see fit. std::map name_2_longname = { {"lev","hybrid level at midpoints (1000*(A+B))"}, @@ -92,6 +101,79 @@ struct LongNames { {"hyam","hybrid A coefficient at layer midpoints"}, {"hybm","hybrid B coefficient at layer midpoints"} }; + + // Create map of longnames, can be added to as developers see fit. + std::map name_2_standardname = { + {"p_mid" , "air_pressure"}, + {"p_mid_at_cldtop" , "air_pressure_at_cloud_top"}, + {"T_2m" , "air_temperature"}, + {"T_mid" , "air_temperature"}, + {"T_mid_at_cldtop" , "air_temperature_at_cloud_top"}, + {"aero_g_sw" , "asymmetry_factor_of_ambient_aerosol_particles"}, + {"pbl_height" , "atmosphere_boundary_layer_thickness"}, + {"precip_liq_surf_mass" , "atmosphere_mass_content_of_liquid_precipitation"}, + {"cldlow" , "cloud_area_fraction"}, + {"cldmed" , "cloud_area_fraction"}, + {"cldhgh" , "cloud_area_fraction"}, + {"cldtot" , "cloud_area_fraction"}, + {"cldfrac_tot_at_cldtop" , "cloud_area_fraction"}, + {"cldfrac_tot" , "cloud_area_fraction_in_atmosphere_layer"}, + {"cldfrac_tot_for_analysis" , "cloud_area_fraction_in_atmosphere_layer"}, + {"cldfrac_rad" , "cloud_area_fraction_in_atmosphere_layer"}, + {"qi" , "cloud_ice_mixing_ratio"}, + {"qc" , "cloud_liquid_water_mixing_ratio"}, + {"U" , "eastward_wind"}, + {"eff_radius_qi" , "effective_radius_of_cloud_ice_particles"}, + {"eff_radius_qc" , "effective_radius_of_cloud_liquid_water_particles"}, + {"eff_radius_qc_at_cldtop" , "effective_radius_of_cloud_liquid_water_particles_at_liquid_water_cloud_top"}, + {"eff_radius_qr" , "effective_radius_of_cloud_rain_particles"}, + {"qv" , "humidity_mixing_ratio"}, + {"cldfrac_ice_at_cldtop" , "ice_cloud_area_fraction"}, + {"cldfrac_ice" , "ice_cloud_area_fraction_in_atmosphere_layer"}, + {"omega" , "lagrangian_tendency_of_air_pressure"}, + {"landfrac" , "land_area_fraction"}, + {"latitude" , "latitude"}, + {"cldfrac_liq_at_cldtop" , "liquid_water_cloud_area_fraction"}, + {"cldfrac_liq" , "liquid_water_cloud_area_fraction_in_atmosphere_layer"}, + {"longitude" , "longitude"}, + {"rainfrac" , "mass_fraction_of_liquid_precipitation_in_air"}, + {"V" , "northward_wind"}, + {"nc" , "number_concentration_of_cloud_liquid_water_particles_in_air"}, + {"cdnc_at_cldtop" , "number_concentration_of_cloud_liquid_water_particles_in_air_at_liquid_water_cloud_top"}, + {"ni" , "number_concentration_of_ice_crystals_in_air"}, + {"aero_tau_sw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, + {"aero_tau_lw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, + {"aero_ssa_sw" , "single_scattering_albedo_in_air_due_to_ambient_aerosol_particles"}, + {"sunlit" , "sunlit_binary_mask"}, + {"ps" , "surface_air_pressure"}, + {"LW_flux_dn_at_model_bot" , "surface_downwelling_longwave_flux_in_air"}, + {"SW_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air"}, + {"SW_clrsky_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air_assuming_clear_sky"}, + {"phis" , "surface_geopotential"}, + {"surf_radiative_T" , "surface_temperature"}, + {"surf_sens_flux" , "surface_upward_sensible_heat_flux"}, + {"SW_flux_dn_at_model_top" , "toa_incoming_shortwave_flux"}, + {"LW_flux_up_at_model_top" , "toa_outgoing_longwave_flux"}, + {"LW_clrsky_flux_up_at_model_top" , "toa_outgoing_longwave_flux_assuming_clear_sky"}, + {"surf_evap" , "water_evapotranspiration_flux"}, + {"AtmosphereDensity" , "air_density"}, + {"PotentialTemperature" , "air_potential_temperature"}, + {"SeaLevelPressure" , "air_pressure_at_mean_sea_level"}, + {"IceWaterPath" , "atmosphere_mass_content_of_cloud_ice"}, + {"LiqWaterPath" , "atmosphere_mass_content_of_cloud_liquid_water"}, + {"VapWaterPath" , "atmosphere_mass_content_of_water_vapor"}, + {"AerosolOpticalDepth550nm" , "atmosphere_optical_thickness_due_to_ambient_aerosol_particles"}, + {"Exner" , "dimensionless_exner_function"}, + {"z_mid" , "geopotential_height"}, + {"geopotential_mid" , "geopotential_height"}, + {"RelativeHumidity" , "relative_humidity"}, + {"surface_upward_latent_heat_flux" , "surface_upward_latent_heat_flux"}, + {"LongwaveCloudForcing" , "toa_longwave_cloud_radiative_effect"}, + {"ShortwaveCloudForcing" , "toa_shortwave_cloud_radiative_effect"}, + {"VirtualTemperature" , "virtual_temperature"}, + {"VaporFlux" , "water_evapotranspiration_flux"}, + {"wind_speed" , "wind_speed"} + }; }; From d44a616970c21cc26bc58b04b272d2c3cb19bb69 Mon Sep 17 00:00:00 2001 From: AaronDonahue Date: Mon, 11 Nov 2024 17:57:56 -0700 Subject: [PATCH 364/366] fix standard names for low/med/hgh cloud area fraction types --- components/eamxx/src/share/io/scream_io_utils.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 796ecd2a003..9eda82e2bbd 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -112,9 +112,9 @@ struct DefaultMetadata { {"aero_g_sw" , "asymmetry_factor_of_ambient_aerosol_particles"}, {"pbl_height" , "atmosphere_boundary_layer_thickness"}, {"precip_liq_surf_mass" , "atmosphere_mass_content_of_liquid_precipitation"}, - {"cldlow" , "cloud_area_fraction"}, - {"cldmed" , "cloud_area_fraction"}, - {"cldhgh" , "cloud_area_fraction"}, + {"cldlow" , "low_type_cloud_area_fraction"}, + {"cldmed" , "medium_type_cloud_area_fraction"}, + {"cldhgh" , "high_type_cloud_area_fraction"}, {"cldtot" , "cloud_area_fraction"}, {"cldfrac_tot_at_cldtop" , "cloud_area_fraction"}, {"cldfrac_tot" , "cloud_area_fraction_in_atmosphere_layer"}, From af3343b7e0681cc0e392d798b7be3ab261ca5467 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 19 Nov 2024 19:40:56 -0700 Subject: [PATCH 365/366] EAMxx: Add aerosols and gases outputs to ensure the microscopy baseline test is sensitive to changes in the interface. --- .../mam/aero_microphys/output.yaml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml b/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml index 388781fb8f0..5af90cac547 100644 --- a/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml +++ b/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml @@ -6,6 +6,62 @@ Fields: Physics: Field Names: - T_mid + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 # To save these fields make sure to turn on # OUTPUT_TRACER_FIELDS #- oxi_fields From 1df5b172c4e2cf671f17858eae701bdf60a8b277 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 20 Nov 2024 15:04:27 -0700 Subject: [PATCH 366/366] Mergify: dismiss reviews when new commits are pushed --- .mergify.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 9c2b8fe13f1..89fcc821e57 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -31,6 +31,12 @@ merge_protections: - check-skipped=cpu-gcc pull_request_rules: + - name: dismiss stale reviews + conditions: + - base=master + actions: + dismiss_reviews: + when: synchronize # Dismiss reviews when synchronize event happens - name: Automatic merge when CI passes and approved conditions: - "label=CI: automerge"