Merge pull request #897 from fvitt/misc_waccmx

cam6_3_133: Hybrid PGF option in the SE dycore for WACCM-x and other misc updates

bld/namelist_files/namelist_defaults_cam.xml

@@ -167,6 +167,7 @@
 <ncdata dyn="se"  hgrid="ne16np4"   nlev="126" waccmx="1" aquaplanet="1">atm/waccm/ic/waccmx_aqua_ne16np4_126L_c191108.nc</ncdata>
 <ncdata dyn="se"  hgrid="ne16np4"   nlev="126" waccmx="1" aquaplanet="1" ionosphere="none">atm/waccm/ic/waccmx4_neutral_aquap_ne16np4_126lev_c200827.nc</ncdata>
 <ncdata dyn="se"  hgrid="ne30np4"   nlev="130" waccmx="1"               >atm/waccm/ic/fx2000_phys-ionos-cpl_ne30_spinup01.cam.i.0002-01-01-00000_c201014.nc</ncdata>
+<ncdata dyn="se"  hgrid="ne30np4" npg="3" nlev="130" waccmx="1"         >atm/waccm/ic/waccmx_ne30pg3_c231005.nc</ncdata>
 
 <ncdata dyn="fv3"   hgrid="C24"  nlev="32"  aquaplanet="1"  ic_ymd="101" >atm/cam/inic/fv3/aqua_0006-01-01_C24_L32_c200625.nc</ncdata>
 <ncdata dyn="fv3"   hgrid="C48"  nlev="32"  aquaplanet="1"  ic_ymd="101" >atm/cam/inic/fv3/aqua_0006-01-01_C48_L32_c200625.nc</ncdata>
@@ -3027,7 +3028,7 @@
 <!-- ================================================================== -->
 
 <se_pgf_formulation>           1</se_pgf_formulation>
-<se_pgf_formulation waccmx="1">2</se_pgf_formulation>
+<se_pgf_formulation waccmx="1">3</se_pgf_formulation>
 <se_ftype> 2 </se_ftype>
 
 <se_large_Courant_incr        > .true.  </se_large_Courant_incr>

bld/namelist_files/namelist_definition.xml

@@ -8174,13 +8174,17 @@ Default: Set by build-namelist.
 </entry>
 
 <entry id="se_pgf_formulation" type="integer" category="se"
-       group="dyn_se_inparm" valid_values="1,2" >
+       group="dyn_se_inparm" valid_values="1,2,3" >
 
   1: Exner version of pressure gradient force (PGF)
   see Appendix A in https://agupubs.onlinelibrary.wiley.com/doi/epdf/10.1029/2022MS003192
 
   2: Traditional pressure gradient formulation (grad p)
 
+  3: Hybrid (formulation 1 where hybm>0 else formulation 2)
+     Use hybrid PGF option for WACCM-x to make WACCM-x consistent with PGF
+     used in CAM in the troposphere and traditional PGF formulation above
+
   Default: Set by build-namelist.
 </entry>
 

cime_config/config_pes.xml

@@ -293,6 +293,27 @@
 	</rootpe>
       </pes>
     </mach>
+    <mach name="derecho">
+      <pes pesize="any" compset="_CAM\d0%WX">
+	<comment>none</comment>
+	<ntasks>
+	  <ntasks_atm>-4</ntasks_atm>
+	  <ntasks_lnd>-4</ntasks_lnd>
+	  <ntasks_rof>-4</ntasks_rof>
+	  <ntasks_ice>-4</ntasks_ice>
+	  <ntasks_ocn>-4</ntasks_ocn>
+	  <ntasks_cpl>-4</ntasks_cpl>
+	</ntasks>
+	<nthrds>
+	  <nthrds_atm>1</nthrds_atm>
+	  <nthrds_lnd>1</nthrds_lnd>
+	  <nthrds_rof>1</nthrds_rof>
+	  <nthrds_ice>1</nthrds_ice>
+	  <nthrds_ocn>1</nthrds_ocn>
+	  <nthrds_cpl>1</nthrds_cpl>
+	</nthrds>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%ne30" >
     <mach name="cheyenne">
@@ -330,6 +351,46 @@
 	</rootpe>
       </pes>
     </mach>
+    <mach name="derecho">
+      <pes pesize="any" compset="_CAM\d0%WX">
+	<comment>none</comment>
+	<ntasks>
+	  <ntasks_atm>-16</ntasks_atm>
+	  <ntasks_lnd>-16</ntasks_lnd>
+	  <ntasks_rof>-16</ntasks_rof>
+	  <ntasks_ice>-16</ntasks_ice>
+	  <ntasks_ocn>-16</ntasks_ocn>
+	  <ntasks_cpl>-16</ntasks_cpl>
+	</ntasks>
+	<nthrds>
+	  <nthrds_atm>1</nthrds_atm>
+	  <nthrds_lnd>1</nthrds_lnd>
+	  <nthrds_rof>1</nthrds_rof>
+	  <nthrds_ice>1</nthrds_ice>
+	  <nthrds_ocn>1</nthrds_ocn>
+	  <nthrds_cpl>1</nthrds_cpl>
+	</nthrds>
+      </pes>
+      <pes pesize="any" compset="_CAM%DEV%MT%CC">
+	<comment>none</comment>
+	<ntasks>
+	  <ntasks_atm>-8</ntasks_atm>
+	  <ntasks_lnd>-8</ntasks_lnd>
+	  <ntasks_rof>-8</ntasks_rof>
+	  <ntasks_ice>-8</ntasks_ice>
+	  <ntasks_ocn>-8</ntasks_ocn>
+	  <ntasks_cpl>-8</ntasks_cpl>
+	</ntasks>
+	<nthrds>
+	  <nthrds_atm>1</nthrds_atm>
+	  <nthrds_lnd>1</nthrds_lnd>
+	  <nthrds_rof>1</nthrds_rof>
+	  <nthrds_ice>1</nthrds_ice>
+	  <nthrds_ocn>1</nthrds_ocn>
+	  <nthrds_cpl>1</nthrds_cpl>
+	</nthrds>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%ne30" >
     <mach name="cheyenne">
@@ -1105,6 +1166,27 @@
 	</rootpe>
       </pes>
     </mach>
+    <mach name="derecho">
+      <pes pesize="any" compset="_CAM\d0%WX">
+	<comment>none</comment>
+	<ntasks>
+	  <ntasks_atm>-8</ntasks_atm>
+	  <ntasks_lnd>-8</ntasks_lnd>
+	  <ntasks_rof>-8</ntasks_rof>
+	  <ntasks_ice>-8</ntasks_ice>
+	  <ntasks_ocn>-8</ntasks_ocn>
+	  <ntasks_cpl>-8</ntasks_cpl>
+	</ntasks>
+	<nthrds>
+	  <nthrds_atm>1</nthrds_atm>
+	  <nthrds_lnd>1</nthrds_lnd>
+	  <nthrds_rof>1</nthrds_rof>
+	  <nthrds_ice>1</nthrds_ice>
+	  <nthrds_ocn>1</nthrds_ocn>
+	  <nthrds_cpl>1</nthrds_cpl>
+	</nthrds>
+      </pes>
+    </mach>
   </grid>
 
   <grid name="a%0.9x1.25">
@@ -1211,6 +1293,27 @@
 	</rootpe>
       </pes>
     </mach>
+    <mach name="derecho">
+      <pes pesize="any" compset="_(CAM50|CAM60)%(CC|CV|CF|WC)">
+	<comment>none</comment>
+	<ntasks>
+	  <ntasks_atm>-8</ntasks_atm>
+	  <ntasks_lnd>-8</ntasks_lnd>
+	  <ntasks_rof>-8</ntasks_rof>
+	  <ntasks_ice>-8</ntasks_ice>
+	  <ntasks_ocn>-8</ntasks_ocn>
+	  <ntasks_cpl>-8</ntasks_cpl>
+	</ntasks>
+	<nthrds>
+	  <nthrds_atm>1</nthrds_atm>
+	  <nthrds_lnd>1</nthrds_lnd>
+	  <nthrds_rof>1</nthrds_rof>
+	  <nthrds_ice>1</nthrds_ice>
+	  <nthrds_ocn>1</nthrds_ocn>
+	  <nthrds_cpl>1</nthrds_cpl>
+	</nthrds>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%0.47x0.63" >
     <mach name="any">

doc/ChangeLog

@@ -1,5 +1,123 @@
 ===============================================================
 
+Tag name: cam6_3_133
+Originator(s): fvitt
+Date: 19 Oct 2023
+One-line Summary: Misc updates for WACCM-X
+Github PR URL: https://github.com/ESCOMP/CAM/pull/897
+
+Purpose of changes (include the issue number and title text for each relevant GitHub issue):
+
+  Introduce a hybrid PGF option in the SE dycore for WACCM-x to be consistent
+  with PGF used in CAM in the troposphere and traditional PGF formulation above
+  to solve stability issues with the SE dycore (issue #896)
+
+  Correction to geometric height diagnostic (issue #681)
+
+  Implements PHIHM history field for waccmx -- High Latitude Electric Potential
+
+  Set reasonable default PE layouts for WACCM(x) and CAMChem on derecho. This
+  solves some of the regression test failures on derecho listed in issue #892
+
+Describe any changes made to build system: N/A
+
+Describe any changes made to the namelist: N/A
+
+List any changes to the defaults for the boundary datasets: N/A
+
+Describe any substantial timing or memory changes: N/a
+
+Code reviewed by: PeterHjortLauritzen cacraigucar nusbaume
+
+List all files eliminated: N/A
+
+List all files added and what they do: N/A
+
+List all existing files that have been modified, and describe the changes:
+
+M       bld/namelist_files/namelist_defaults_cam.xml
+ - default IC for ne30pg3 L130 waccmx
+
+M       bld/namelist_files/namelist_definition.xml
+ - add hybrid option for PGF formulation used by waccmx
+
+M       cime_config/config_pes.xml
+ - default derecho PE layouts for
+    ne16, ne30 waccmx
+    f09 waccmx, waccm, and camchem
+
+M       src/control/cam_history.F90
+ - deallocate arrays at the end of write_restart_history
+
+M       src/dynamics/se/dycore/global_norms_mod.F90
+ - for waccmx set umax to 800 m/s used in stability assessment
+
+M       src/dynamics/se/dycore/prim_advance_mod.F90
+ - add hybrid option for PGF formulation used by waccmx
+
+M       src/ionosphere/waccmx/edyn_init.F90
+M       src/ionosphere/waccmx/edynamo.F90
+ - add PHIHM history field diagnostic
+
+M       src/ionosphere/waccmx/ionosphere_interface.F90
+ - correction to Z3GM geometric height diagnostic
+ - replace "/gravit" with "*rga" in height calc
+
+M       src/physics/cam/vertical_diffusion.F90
+ - raise the lid of eddy diffusion for waccmx (ntop_eddy_pres=1.e-7 Pa)
+
+If there were any failures reported from running test_driver.sh on any test
+platform, and checkin with these failures has been OK'd by the gatekeeper,
+then copy the lines from the td.*.status files for the failed tests to the
+appropriate machine below.  All failed tests must be justified.
+
+cheyenne/intel/aux_cam:
+
+  ERP_Ln9_Vnuopc.C96_C96_mg17.F2000climo.cheyenne_intel.cam-outfrq9s_mg3 (Overall: FAIL) details:
+    FAIL ERP_Ln9_Vnuopc.C96_C96_mg17.F2000climo.cheyenne_intel.cam-outfrq9s_mg3 MODEL_BUILD time=3
+  ERP_Ln9_Vnuopc.f09_f09_mg17.FCSD_HCO.cheyenne_intel.cam-outfrq9s (Overall: FAIL) details:
+    FAIL ERP_Ln9_Vnuopc.f09_f09_mg17.FCSD_HCO.cheyenne_intel.cam-outfrq9s COMPARE_base_rest
+    FAIL ERP_Ln9_Vnuopc.f09_f09_mg17.FCSD_HCO.cheyenne_intel.cam-outfrq9s BASELINE /glade/p/cesm/amwg/cesm_baselines/cam6_3_132: DIFF
+  SMS_Lh12_Vnuopc.f09_f09_mg17.FCSD_HCO.cheyenne_intel.cam-outfrq3h (Overall: DIFF) details:
+    FAIL SMS_Lh12_Vnuopc.f09_f09_mg17.FCSD_HCO.cheyenne_intel.cam-outfrq3h BASELINE /glade/p/cesm/amwg/cesm_baselines/cam6_3_132: DIFF
+ - pre-existing failure
+
+  ERC_D_Ln9_Vnuopc.f19_f19_mg17.QPX2000.cheyenne_intel.cam-outfrq3s (Overall: DIFF) details:
+    FAIL ERC_D_Ln9_Vnuopc.f19_f19_mg17.QPX2000.cheyenne_intel.cam-outfrq3s BASELINE /glade/p/cesm/amwg/cesm_baselines/cam6_3_132: DIFF
+  ERS_Ln9_Vnuopc.f09_f09_mg17.FX2000.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details:
+    FAIL ERS_Ln9_Vnuopc.f09_f09_mg17.FX2000.cheyenne_intel.cam-outfrq9s BASELINE /glade/p/cesm/amwg/cesm_baselines/cam6_3_132: DIFF
+  ERS_Ln9_Vnuopc.f19_f19_mg17.FXSD.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details:
+    FAIL ERS_Ln9_Vnuopc.f19_f19_mg17.FXSD.cheyenne_intel.cam-outfrq9s BASELINE /glade/p/cesm/amwg/cesm_baselines/cam6_3_132: DIFF
+  SMS_D_Ln9_Vnuopc.f19_f19_mg17.FXHIST.cheyenne_intel.cam-outfrq9s_amie (Overall: DIFF) details:
+    FAIL SMS_D_Ln9_Vnuopc.f19_f19_mg17.FXHIST.cheyenne_intel.cam-outfrq9s_amie BASELINE /glade/p/cesm/amwg/cesm_baselines/cam6_3_132: DIFF
+  SMS_D_Ln9_Vnuopc.ne16_ne16_mg17.FX2000.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details:
+    FAIL SMS_D_Ln9_Vnuopc.ne16_ne16_mg17.FX2000.cheyenne_intel.cam-outfrq9s NLCOMP
+    FAIL SMS_D_Ln9_Vnuopc.ne16_ne16_mg17.FX2000.cheyenne_intel.cam-outfrq9s BASELINE /glade/p/cesm/amwg/cesm_baselines/cam6_3_132: DIFF
+  SMS_D_Ln9_Vnuopc.ne16_ne16_mg17.QPX2000.cheyenne_intel.cam-outfrq9s (Overall: DIFF) details:
+    FAIL SMS_D_Ln9_Vnuopc.ne16_ne16_mg17.QPX2000.cheyenne_intel.cam-outfrq9s NLCOMP
+    FAIL SMS_D_Ln9_Vnuopc.ne16_ne16_mg17.QPX2000.cheyenne_intel.cam-outfrq9s BASELINE /glade/p/cesm/amwg/cesm_baselines/cam6_3_132: DIFF
+ - expected failures
+
+izumi/nag/aux_cam:
+
+  DAE_Vnuopc.f45_f45_mg37.FHS94.izumi_nag.cam-dae (Overall: FAIL) details:
+    FAIL DAE_Vnuopc.f45_f45_mg37.FHS94.izumi_nag.cam-dae RUN time=205
+ - pre-existing failure
+
+  SMS_D_Ln3_Vnuopc.ne5pg3_ne5pg3_mg37.QPX2000.izumi_nag.cam-outfrq3s (Overall: DIFF) details:
+    FAIL SMS_D_Ln3_Vnuopc.ne5pg3_ne5pg3_mg37.QPX2000.izumi_nag.cam-outfrq3s NLCOMP
+    FAIL SMS_D_Ln3_Vnuopc.ne5pg3_ne5pg3_mg37.QPX2000.izumi_nag.cam-outfrq3s BASELINE /fs/cgd/csm/models/atm/cam/pretag_bl/cam6_3_132_nag: DIFF
+ - expected failure
+
+izumi/gnu/aux_cam: All PASS
+
+Summarize any changes to answers:
+  larger than roundoff for SE WACCMX, roundoff for FV WACCMX,
+  otherwise bit-for-bit unchanged
+
+===============================================================
+===============================================================
+
 Tag name:cam6_3_132
 Originator(s): cacraig, adamher, Thomas Toniazzo, hannay
 Date: Oct 18, 2023

src/control/cam_history.F90

@@ -1747,7 +1747,8 @@ subroutine write_restart_history ( File, &
       ierr = pio_put_var(File, mdimname_desc, start, hist_coord_name(f))
     end do
 
-    deallocate(xyfill, allmdims)
+    deallocate(xyfill, allmdims, is_subcol, interp_output, restarthistory_tape)
+
     return
 
   end subroutine write_restart_history
@@ -5019,19 +5020,19 @@ subroutine h_global (f, t)
     !
     type (dim_index_2d)     :: dimind    ! 2-D dimension index
     integer                 :: ie        ! dim3 index
-    integer                 :: count     ! tmp index 
+    integer                 :: count     ! tmp index
     integer                 :: i1        ! dim1 index
     integer                 :: j1        ! dim2 index
     integer                 :: fdims(3)  ! array shape
-    integer                 :: begdim1,enddim1,begdim2,enddim2,begdim3,enddim3        ! 
+    integer                 :: begdim1,enddim1,begdim2,enddim2,begdim3,enddim3        !
     real(r8)                :: globalsum(1) ! globalsum
     real(r8), allocatable   :: globalarr(:) ! globalarr values for this pe
-        
+
     call t_startf ('h_global')
 
     ! wbuf contains the area weighting for this field decomposition
     if (associated(tape(t)%hlist(f)%wbuf) ) then
-       
+
        begdim1    =  tape(t)%hlist(f)%field%begdim1
        enddim1    =  tape(t)%hlist(f)%field%enddim1
        fdims(1)   =  enddim1 - begdim1 + 1
@@ -5041,7 +5042,7 @@ subroutine h_global (f, t)
        begdim3    =  tape(t)%hlist(f)%field%begdim3
        enddim3    =  tape(t)%hlist(f)%field%enddim3
        fdims(3)   =  enddim3 - begdim3 + 1
-       
+
        allocate(globalarr(fdims(1)*fdims(2)*fdims(3)))
        count=0
        globalarr=0._r8
@@ -5098,7 +5099,7 @@ subroutine h_field_op (f, t)
 
     do c = begdim3, enddim3
       dimind = tape(t)%hlist(f)%field%get_dims(c)
-      if (trim(optype) == 'dif') then 
+      if (trim(optype) == 'dif') then
          tape(t)%hlist(f)%hbuf(dimind%beg1:dimind%end1,dimind%beg2:dimind%end2,c) = &
          tape(t)%hlist(f1)%hbuf(dimind%beg1:dimind%end1,dimind%beg2:dimind%end2,c) - &
          tape(t)%hlist(f2)%hbuf(dimind%beg1:dimind%end1,dimind%beg2:dimind%end2,c)
@@ -5592,7 +5593,7 @@ subroutine wshist (rgnht_in)
           if(.not. restart) then
              !$OMP PARALLEL DO PRIVATE (F)
              do f=1,nflds(t)
-                ! Normalize all non composed fields, composed fields are calculated next using the normalized components 
+                ! Normalize all non composed fields, composed fields are calculated next using the normalized components
                 if (tape(t)%hlist(f)%avgflag /= 'I'.and..not.tape(t)%hlist(f)%field%is_composed()) then
                    call h_normalize (f, t)
                 end if
@@ -5916,7 +5917,7 @@ subroutine addfld_nd(fname, dimnames, avgflag, units, long_name,            &
 
     if (present(optype)) then
        ! make sure optype is "sum" or "dif"
-       if (.not.(trim(optype) == 'dif' .or. trim(optype) == 'sum')) then 
+       if (.not.(trim(optype) == 'dif' .or. trim(optype) == 'sum')) then
           write(errormsg, '(2a)')': Fatal : optype must be "sum" or "dif" not ',trim(optype)
           call endrun (trim(subname)//errormsg)
        end if

src/dynamics/se/dycore/global_norms_mod.F90

@@ -737,9 +737,12 @@ subroutine print_cfl(elem,hybrid,nets,nete,dtnu,ptop,pmid,&
     !
     if (ptop>100.0_r8) then
       umax = 120.0_r8
-    else
+    else if (ptop>10.0_r8) then
       umax = 400.0_r8
+    else
+      umax = 800.0_r8
     end if
+
     ugw = 342.0_r8 !max gravity wave speed
 
     dt_max_adv             = S_rk/(umax*max_normDinv*lambda_max*ra)