From e543f754b28951030a660cc670b799be79e48174 Mon Sep 17 00:00:00 2001
From: noel <ndkeen@lbl.gov>
Date: Fri, 19 Jan 2024 09:14:37 -0800
Subject: [PATCH] updates to maint-2.1 for pm-cpu that were made to master

---
 cime_config/allactive/config_pesall.xml       | 40 +++++-------------
 cime_config/machines/config_batch.xml         |  4 +-
 cime_config/machines/config_machines.xml      | 41 ++++++++++++++-----
 .../allactive/wcprodssp/user_nl_elm           |  1 +
 components/eam/cime_config/config_pes.xml     | 14 ++-----
 .../mpas-ocean/cime_config/config_pes.xml     |  7 +---
 6 files changed, 50 insertions(+), 57 deletions(-)

diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml
index 6f7bac500281..ac05222224ff 100644
--- a/cime_config/allactive/config_pesall.xml
+++ b/cime_config/allactive/config_pesall.xml
@@ -265,7 +265,7 @@
   <grid name="a%ne30np4">
     <mach name="pm-cpu|alvarez">
       <pes compset="JRA_ELM.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="any">
-        <comment>"pm-cpu 4 nodes, 256 partition, 128x1, c8"</comment>
+        <comment>"pm-cpu 4 nodes, 256 partition, 128x1"</comment>
         <ntasks>
           <ntasks_atm>-4</ntasks_atm>
           <ntasks_lnd>-4</ntasks_lnd>
@@ -274,11 +274,8 @@
           <ntasks_ocn>-4</ntasks_ocn>
           <ntasks_glc>-1</ntasks_glc>
           <ntasks_wav>-1</ntasks_wav>
-          <ntasks_cpl>64</ntasks_cpl>
+          <ntasks_cpl>-4</ntasks_cpl>
         </ntasks>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
     <mach name="gcp12">
@@ -552,11 +549,11 @@
   <grid name="a%ne120np4">
     <mach name="pm-cpu|alvarez">
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+SWAV.*" pesize="any">
-        <comment>ne120-wcycl on 42 nodes 128x1c8 ~0.7 sypd</comment>
+        <comment>ne120-wcycl on 42 nodes 128x1 ~0.7 sypd</comment>
         <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>3072</ntasks_atm>
-          <ntasks_cpl>384</ntasks_cpl>
+          <ntasks_cpl>3072</ntasks_cpl>
           <ntasks_ice>3072</ntasks_ice>
           <ntasks_lnd>2560</ntasks_lnd>
           <ntasks_rof>512</ntasks_rof>
@@ -584,9 +581,6 @@
           <rootpe_glc>0</rootpe_glc>
           <rootpe_wav>0</rootpe_wav>
         </rootpe>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
     <mach name="theta">
@@ -1252,7 +1246,7 @@
     </mach>
     <mach name="pm-cpu|alvarez">
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="any">
-        <comment> -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 7 nodes, 128x1 c8 </comment>
+        <comment> -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 7 nodes, 128x1 </comment>
         <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>640</ntasks_atm>
@@ -1260,7 +1254,7 @@
           <ntasks_rof>640</ntasks_rof>
           <ntasks_ice>640</ntasks_ice>
           <ntasks_ocn>256</ntasks_ocn>
-          <ntasks_cpl>80</ntasks_cpl>
+          <ntasks_cpl>640</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -1278,9 +1272,6 @@
           <rootpe_ocn>640</rootpe_ocn>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="L">
         <comment> -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 58 nodes, ~20 sypd</comment>
@@ -1291,7 +1282,7 @@
           <ntasks_rof>256</ntasks_rof>
           <ntasks_ice>5248</ntasks_ice>
           <ntasks_ocn>1920</ntasks_ocn>
-          <ntasks_cpl>688</ntasks_cpl>
+          <ntasks_cpl>5504</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -1309,9 +1300,6 @@
           <rootpe_ocn>5504</rootpe_ocn>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
   </grid>
@@ -1757,7 +1745,7 @@
     </mach>
     <mach name="pm-cpu|alvarez">
       <pes compset="any" pesize="any">
-        <comment>"pm-cpu ne30np4 and ne30np4.pg2 2 nodes 1 thread, 128x1 c8"</comment>
+        <comment>"pm-cpu ne30np4 and ne30np4.pg2 2 nodes 1 thread, 128x1"</comment>
         <ntasks>
           <ntasks_atm>-2</ntasks_atm>
           <ntasks_lnd>-2</ntasks_lnd>
@@ -1766,11 +1754,8 @@
           <ntasks_ocn>-2</ntasks_ocn>
           <ntasks_glc>-2</ntasks_glc>
           <ntasks_wav>-2</ntasks_wav>
-          <ntasks_cpl>32</ntasks_cpl>
+          <ntasks_cpl>-2</ntasks_cpl>
         </ntasks>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
     <mach name="crusher-scream-gpu">
@@ -2072,14 +2057,14 @@
     </mach>
     <mach name="pm-cpu|alvarez">
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="any">
-        <comment> 8 nodes, 128x1 c8</comment>
+        <comment> 8 nodes, 128x1</comment>
         <ntasks>
           <ntasks_atm>640</ntasks_atm>
           <ntasks_lnd>640</ntasks_lnd>
           <ntasks_rof>640</ntasks_rof>
           <ntasks_ice>640</ntasks_ice>
           <ntasks_ocn>384</ntasks_ocn>
-          <ntasks_cpl>80</ntasks_cpl>
+          <ntasks_cpl>640</ntasks_cpl>
         </ntasks>
         <rootpe>
           <rootpe_atm>0</rootpe_atm>
@@ -2097,9 +2082,6 @@
           <nthrds_ocn>1</nthrds_ocn>
           <nthrds_cpl>1</nthrds_cpl>
         </nthrds>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
   </grid>
diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml
index 714193598db5..6adc14af40cd 100644
--- a/cime_config/machines/config_batch.xml
+++ b/cime_config/machines/config_batch.xml
@@ -384,8 +384,8 @@
     </directives>
     <queues>
       <!-- Note: walltime is not the max walltime, but the default - see NERSC docs for Q limits, https://docs.nersc.gov/jobs/policy/ -->
-      <queue walltimemax="12:00:00" nodemax="4096" default="true">regular</queue>
-      <queue walltimemax="12:00:00" nodemax="4096" strict="true">preempt</queue>
+      <queue walltimemax="12:00:00" nodemax="3072" default="true">regular</queue>
+      <queue walltimemax="12:00:00" nodemax="3072" strict="true">preempt</queue>
       <queue walltimemax="00:30:00" nodemax="8" strict="true">debug</queue>
     </queues>
   </batch_system>
diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
index aa6e8d0b34cf..ce6bb238fba5 100644
--- a/cime_config/machines/config_machines.xml
+++ b/cime_config/machines/config_machines.xml
@@ -208,6 +208,8 @@
         <command name="unload">cray-hdf5-parallel</command>
         <command name="unload">cray-netcdf-hdf5parallel</command>
         <command name="unload">cray-parallel-netcdf</command>
+        <command name="unload">cray-netcdf</command>
+        <command name="unload">cray-hdf5</command>
         <command name="unload">PrgEnv-gnu</command>
         <command name="unload">PrgEnv-intel</command>
         <command name="unload">PrgEnv-nvidia</command>
@@ -215,7 +217,10 @@
         <command name="unload">PrgEnv-aocc</command>
         <command name="unload">intel</command>
         <command name="unload">intel-oneapi</command>
+        <command name="unload">nvidia</command>
+        <command name="unload">aocc</command>
         <command name="unload">cudatoolkit</command>
+        <command name="unload">climate-utils</command>
         <command name="unload">craype-accel-nvidia80</command>
         <command name="unload">craype-accel-host</command>
         <command name="unload">perftools-base</command>
@@ -242,7 +247,7 @@
 
       <modules compiler="amdclang">
         <command name="load">PrgEnv-aocc</command>
-        <command name="load">aocc/3.2.0</command>
+        <command name="load">aocc/4.0.0</command>
         <command name="load">cray-libsci/23.02.1.1</command>
       </modules>
 
@@ -272,6 +277,8 @@
       <env name="PERL5LIB">/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch</env>
       <env name="FI_CXI_RX_MATCH_MODE">software</env>
       <env name="MPICH_COLL_SYNC">MPI_Bcast</env>
+      <env name="NETCDF_PATH">$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}</env>
+      <env name="PNETCDF_PATH">$ENV{CRAY_PARALLEL_NETCDF_PREFIX}</env>
     </environment_variables>
     <resource_limits>
       <resource name="RLIMIT_STACK">-1</resource>
@@ -331,9 +338,19 @@
         <command name="unload">cray-hdf5-parallel</command>
         <command name="unload">cray-netcdf-hdf5parallel</command>
         <command name="unload">cray-parallel-netcdf</command>
+        <command name="unload">cray-netcdf</command>
+        <command name="unload">cray-hdf5</command>
         <command name="unload">PrgEnv-gnu</command>
+        <command name="unload">PrgEnv-intel</command>
         <command name="unload">PrgEnv-nvidia</command>
+        <command name="unload">PrgEnv-cray</command>
+        <command name="unload">PrgEnv-aocc</command>
+        <command name="unload">intel</command>
+        <command name="unload">intel-oneapi</command>
+        <command name="unload">nvidia</command>
+        <command name="unload">aocc</command>
         <command name="unload">cudatoolkit</command>
+        <command name="unload">climate-utils</command>
         <command name="unload">craype-accel-nvidia80</command>
         <command name="unload">craype-accel-host</command>
         <command name="unload">perftools-base</command>
@@ -348,16 +365,16 @@
 
       <modules compiler="nvidia.*">
         <command name="load">PrgEnv-nvidia</command>
-        <command name="load">nvidia/22.5</command>
+        <command name="load">nvidia/22.7</command>
       </modules>
 
       <modules compiler="gnugpu">
-        <command name="load">cudatoolkit/11.5</command>
+        <command name="load">cudatoolkit/11.7</command>
         <command name="load">craype-accel-nvidia80</command>
       </modules>
 
       <modules compiler="nvidiagpu">
-        <command name="load">cudatoolkit/11.5</command>
+        <command name="load">cudatoolkit/11.7</command>
         <command name="load">craype-accel-nvidia80</command>
       </modules>
 
@@ -370,13 +387,13 @@
       </modules>
 
       <modules>
-        <command name="load">cray-libsci</command>
-        <command name="load">craype</command>
-        <command name="load">cray-mpich/8.1.22</command>
-        <command name="load">cray-hdf5-parallel/1.12.2.1</command>
-        <command name="load">cray-netcdf-hdf5parallel/4.9.0.1</command>
-        <command name="load">cray-parallel-netcdf/1.12.3.1</command>
-        <command name="load">cmake/3.22.0</command>
+        <command name="load">cray-libsci/23.02.1.1</command>
+        <command name="load">craype/2.7.20</command>
+        <command name="load">cray-mpich/8.1.25</command>
+        <command name="load">cray-hdf5-parallel/1.12.2.3</command>
+        <command name="load">cray-netcdf-hdf5parallel/4.9.0.3</command>
+        <command name="load">cray-parallel-netcdf/1.12.3.3</command>
+        <command name="load">cmake/3.24.3</command>
       </modules>
     </module_system>
 
@@ -393,6 +410,8 @@
       <env name="HDF5_USE_FILE_LOCKING">FALSE</env>
       <env name="PERL5LIB">/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch</env>
       <env name="MPICH_COLL_SYNC">MPI_Bcast</env>
+      <env name="NETCDF_PATH">$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}</env>
+      <env name="PNETCDF_PATH">$ENV{CRAY_PARALLEL_NETCDF_PREFIX}</env>
     </environment_variables>
     <environment_variables compiler="gnugpu">
       <env name="MPICH_GPU_SUPPORT_ENABLED">1</env>
diff --git a/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm
index 21db35a3f4d2..e7da43000f43 100644
--- a/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm
+++ b/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm
@@ -2,6 +2,7 @@
 
  CHECK_FINIDAT_FSURDAT_CONSISTENCY       = .false.
 
+ finidat = "$DIN_LOC_ROOT/e3sm_init/V2.SSP370_SSP585.ne30pg2_EC30to60E2r2/v2.LR.historical_0101/2015-01-01-00000/v2.LR.historical_0101.elm.r.noNaN.2015-01-01-00000.nc"
  hist_dov2xy = .true.,.true.
  hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA'
  hist_mfilt = 1,365
diff --git a/components/eam/cime_config/config_pes.xml b/components/eam/cime_config/config_pes.xml
index 26ab65818dc3..5625c480f960 100644
--- a/components/eam/cime_config/config_pes.xml
+++ b/components/eam/cime_config/config_pes.xml
@@ -811,7 +811,7 @@
       <!--Pes setting: grid          is a%ne30np4_l%ne30np4_oi%oEC60to30v3_r%r05_g%null_w%null_z%null_m%oEC60to30v3
           Pes setting: compset       is 2010_EAM%CMIP6_ELM%SPBC_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV_SIAC_SESP -->
       <pes compset=".*EAM.+ELM.+DOCN" pesize="any">
-        <comment> -compset A_WCYCL* -res ne30pg2_oECv3 without MPASO on 4 nodes, 128x1 c8 </comment>
+        <comment> -compset A_WCYCL* -res ne30pg2_oECv3 without MPASO on 4 nodes, 128x1 </comment>
         <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>512</ntasks_atm>
@@ -819,11 +819,8 @@
           <ntasks_rof>512</ntasks_rof>
           <ntasks_ice>512</ntasks_ice>
           <ntasks_ocn>512</ntasks_ocn>
-          <ntasks_cpl>64</ntasks_cpl>
+          <ntasks_cpl>512</ntasks_cpl>
         </ntasks>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
   </grid>
@@ -1114,7 +1111,7 @@
   <grid name="a%ne120np4">
     <mach name="pm-cpu|alvarez">
       <pes compset=".*EAM.+ELM.+MPASSI.+DOCN.+SGLC.+SWAV.*" pesize="any">
-        <comment>pm-cpu ne120pg2 F-compset with MPASSI on 43 nodes 128x1c8 1.3 sypd</comment>
+        <comment>pm-cpu ne120pg2 F-compset with MPASSI on 43 nodes 128x1 1.3 sypd</comment>
         <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>5504</ntasks_atm>
@@ -1124,7 +1121,7 @@
           <ntasks_ocn>5504</ntasks_ocn>
           <ntasks_glc>64</ntasks_glc>
           <ntasks_wav>64</ntasks_wav>
-          <ntasks_cpl>688</ntasks_cpl>
+          <ntasks_cpl>5504</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -1136,9 +1133,6 @@
           <nthrds_wav>1</nthrds_wav>
           <nthrds_cpl>1</nthrds_cpl>
         </nthrds>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
     <mach name="theta">
diff --git a/components/mpas-ocean/cime_config/config_pes.xml b/components/mpas-ocean/cime_config/config_pes.xml
index 0ede99f8b83a..a46c39d2c39f 100644
--- a/components/mpas-ocean/cime_config/config_pes.xml
+++ b/components/mpas-ocean/cime_config/config_pes.xml
@@ -428,7 +428,7 @@
     </mach>
     <mach name="pm-cpu|alvarez">
       <pes compset="DATM.+MPASO.+SWAV" pesize="any">
-        <comment>mpas-ocean: SO RRM, compset=DATM+MPASO, 8 nodes, 128x1c8 ~3.3 sypd</comment>
+        <comment>mpas-ocean: SO RRM, compset=DATM+MPASO, 8 nodes, 128x1 ~3.3 sypd</comment>
         <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>1024</ntasks_atm>
@@ -437,7 +437,7 @@
           <ntasks_ice>1024</ntasks_ice>
           <ntasks_ocn>1024</ntasks_ocn>
           <ntasks_glc>1</ntasks_glc>
-          <ntasks_cpl>128</ntasks_cpl>
+          <ntasks_cpl>1024</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -458,9 +458,6 @@
           <rootpe_glc>0</rootpe_glc>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
   </grid>