From 601d6f43c2ce145078db8057c1fb1f90445ebc61 Mon Sep 17 00:00:00 2001
From: noel <ndkeen@lbl.gov>
Date: Fri, 19 Jan 2024 09:26:33 -0800
Subject: [PATCH] updates for maint-2.0 on pm-cpu to match master

---
 cime_config/allactive/config_pesall.xml   | 40 +++++++----------------
 cime_config/machines/config_machines.xml  | 23 +++++++++++--
 components/eam/cime_config/config_pes.xml | 15 +++------
 3 files changed, 36 insertions(+), 42 deletions(-)

diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml
index 23711c845e96..ebb72ffe2519 100644
--- a/cime_config/allactive/config_pesall.xml
+++ b/cime_config/allactive/config_pesall.xml
@@ -265,7 +265,7 @@
   <grid name="a%ne30np4">
     <mach name="pm-cpu|alvarez">
       <pes compset="JRA_ELM.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="any">
-        <comment>"pm-cpu 4 nodes, 256 partition, 128x1, c8"</comment>
+        <comment>"pm-cpu 4 nodes, 256 partition, 128x1"</comment>
         <ntasks>
           <ntasks_atm>-4</ntasks_atm>
           <ntasks_lnd>-4</ntasks_lnd>
@@ -274,11 +274,8 @@
           <ntasks_ocn>-4</ntasks_ocn>
           <ntasks_glc>-1</ntasks_glc>
           <ntasks_wav>-1</ntasks_wav>
-          <ntasks_cpl>64</ntasks_cpl>
+          <ntasks_cpl>-4</ntasks_cpl>
         </ntasks>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
     <mach name="gcp12">
@@ -552,11 +549,11 @@
   <grid name="a%ne120np4">
     <mach name="pm-cpu|alvarez">
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+SWAV.*" pesize="any">
-        <comment>ne120-wcycl on 42 nodes 128x1c8 ~0.7 sypd</comment>
+        <comment>ne120-wcycl on 42 nodes 128x1 ~0.7 sypd</comment>
         <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>3072</ntasks_atm>
-          <ntasks_cpl>384</ntasks_cpl>
+          <ntasks_cpl>3072</ntasks_cpl>
           <ntasks_ice>3072</ntasks_ice>
           <ntasks_lnd>2560</ntasks_lnd>
           <ntasks_rof>512</ntasks_rof>
@@ -584,9 +581,6 @@
           <rootpe_glc>0</rootpe_glc>
           <rootpe_wav>0</rootpe_wav>
         </rootpe>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
     <mach name="theta">
@@ -1217,7 +1211,7 @@
     </mach>
     <mach name="pm-cpu|alvarez">
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="any">
-        <comment> -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 7 nodes, 128x1 c8 </comment>
+        <comment> -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 7 nodes, 128x1 </comment>
         <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>640</ntasks_atm>
@@ -1225,7 +1219,7 @@
           <ntasks_rof>640</ntasks_rof>
           <ntasks_ice>640</ntasks_ice>
           <ntasks_ocn>256</ntasks_ocn>
-          <ntasks_cpl>80</ntasks_cpl>
+          <ntasks_cpl>640</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -1243,9 +1237,6 @@
           <rootpe_ocn>640</rootpe_ocn>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="L">
         <comment> -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 58 nodes, ~20 sypd</comment>
@@ -1256,7 +1247,7 @@
           <ntasks_rof>256</ntasks_rof>
           <ntasks_ice>5248</ntasks_ice>
           <ntasks_ocn>1920</ntasks_ocn>
-          <ntasks_cpl>688</ntasks_cpl>
+          <ntasks_cpl>5504</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -1274,9 +1265,6 @@
           <rootpe_ocn>5504</rootpe_ocn>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
   </grid>
@@ -1697,7 +1685,7 @@
     </mach>
     <mach name="pm-cpu|alvarez">
       <pes compset="any" pesize="any">
-        <comment>"pm-cpu ne30np4 and ne30np4.pg2 2 nodes 1 thread, 128x1 c8"</comment>
+        <comment>"pm-cpu ne30np4 and ne30np4.pg2 2 nodes 1 thread, 128x1"</comment>
         <ntasks>
           <ntasks_atm>-2</ntasks_atm>
           <ntasks_lnd>-2</ntasks_lnd>
@@ -1706,11 +1694,8 @@
           <ntasks_ocn>-2</ntasks_ocn>
           <ntasks_glc>-2</ntasks_glc>
           <ntasks_wav>-2</ntasks_wav>
-          <ntasks_cpl>32</ntasks_cpl>
+          <ntasks_cpl>-2</ntasks_cpl>
         </ntasks>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
     <mach name="crusher-scream-gpu">
@@ -2012,14 +1997,14 @@
     </mach>
     <mach name="pm-cpu|alvarez">
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="any">
-        <comment> 8 nodes, 128x1 c8</comment>
+        <comment> 8 nodes, 128x1</comment>
         <ntasks>
           <ntasks_atm>640</ntasks_atm>
           <ntasks_lnd>640</ntasks_lnd>
           <ntasks_rof>640</ntasks_rof>
           <ntasks_ice>640</ntasks_ice>
           <ntasks_ocn>384</ntasks_ocn>
-          <ntasks_cpl>80</ntasks_cpl>
+          <ntasks_cpl>640</ntasks_cpl>
         </ntasks>
         <rootpe>
           <rootpe_atm>0</rootpe_atm>
@@ -2037,9 +2022,6 @@
           <nthrds_ocn>1</nthrds_ocn>
           <nthrds_cpl>1</nthrds_cpl>
         </nthrds>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
   </grid>
diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
index 541d6b9a35be..cc823c8f542d 100644
--- a/cime_config/machines/config_machines.xml
+++ b/cime_config/machines/config_machines.xml
@@ -104,6 +104,8 @@
         <command name="unload">cray-hdf5-parallel</command>
         <command name="unload">cray-netcdf-hdf5parallel</command>
         <command name="unload">cray-parallel-netcdf</command>
+        <command name="unload">cray-netcdf</command>
+        <command name="unload">cray-hdf5</command>
         <command name="unload">PrgEnv-gnu</command>
         <command name="unload">PrgEnv-intel</command>
         <command name="unload">PrgEnv-nvidia</command>
@@ -111,7 +113,10 @@
         <command name="unload">PrgEnv-aocc</command>
         <command name="unload">intel</command>
         <command name="unload">intel-oneapi</command>
+        <command name="unload">nvidia</command>
+        <command name="unload">aocc</command>
         <command name="unload">cudatoolkit</command>
+        <command name="unload">climate-utils</command>
         <command name="unload">craype-accel-nvidia80</command>
         <command name="unload">craype-accel-host</command>
         <command name="unload">perftools-base</command>
@@ -138,7 +143,7 @@
 
       <modules compiler="amdclang">
         <command name="load">PrgEnv-aocc</command>
-        <command name="load">aocc/3.2.0</command>
+        <command name="load">aocc/4.0.0</command>
         <command name="load">cray-libsci/23.02.1.1</command>
       </modules>
 
@@ -168,6 +173,8 @@
       <env name="PERL5LIB">/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch</env>
       <env name="FI_CXI_RX_MATCH_MODE">software</env>
       <env name="MPICH_COLL_SYNC">MPI_Bcast</env>
+      <env name="NETCDF_PATH">$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}</env>
+      <env name="PNETCDF_PATH">$ENV{CRAY_PARALLEL_NETCDF_PREFIX}</env>
     </environment_variables>
     <resource_limits>
       <resource name="RLIMIT_STACK">-1</resource>
@@ -227,11 +234,19 @@
         <command name="unload">cray-hdf5-parallel</command>
         <command name="unload">cray-netcdf-hdf5parallel</command>
         <command name="unload">cray-parallel-netcdf</command>
+        <command name="unload">cray-netcdf</command>
+        <command name="unload">cray-hdf5</command>
         <command name="unload">PrgEnv-gnu</command>
+        <command name="unload">PrgEnv-intel</command>
         <command name="unload">PrgEnv-nvidia</command>
         <command name="unload">PrgEnv-cray</command>
         <command name="unload">PrgEnv-aocc</command>
+        <command name="unload">intel</command>
+        <command name="unload">intel-oneapi</command>
+        <command name="unload">nvidia</command>
+        <command name="unload">aocc</command>
         <command name="unload">cudatoolkit</command>
+        <command name="unload">climate-utils</command>
         <command name="unload">craype-accel-nvidia80</command>
         <command name="unload">craype-accel-host</command>
         <command name="unload">perftools-base</command>
@@ -250,12 +265,12 @@
       </modules>
 
       <modules compiler="gnugpu">
-        <command name="load">cudatoolkit/11.5</command>
+        <command name="load">cudatoolkit/11.7</command>
         <command name="load">craype-accel-nvidia80</command>
       </modules>
 
       <modules compiler="nvidiagpu">
-        <command name="load">cudatoolkit/11.5</command>
+        <command name="load">cudatoolkit/11.7</command>
         <command name="load">craype-accel-nvidia80</command>
       </modules>
 
@@ -291,6 +306,8 @@
       <env name="HDF5_USE_FILE_LOCKING">FALSE</env>
       <env name="PERL5LIB">/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch</env>
       <env name="MPICH_COLL_SYNC">MPI_Bcast</env>
+      <env name="NETCDF_PATH">$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}</env>
+      <env name="PNETCDF_PATH">$ENV{CRAY_PARALLEL_NETCDF_PREFIX}</env>
     </environment_variables>
     <environment_variables compiler="gnugpu">
       <env name="MPICH_GPU_SUPPORT_ENABLED">1</env>
diff --git a/components/eam/cime_config/config_pes.xml b/components/eam/cime_config/config_pes.xml
index 09a2475e9665..0c701ef0dd0b 100644
--- a/components/eam/cime_config/config_pes.xml
+++ b/components/eam/cime_config/config_pes.xml
@@ -217,6 +217,7 @@
     </mach>
     <!-- end machine-specific generic defaults -->
   </grid>
+  <!-- ne4 PEs -->
   <grid name="a%ne4np4.pg2">
     <mach name="any">
       <pes compset="any" pesize="any">
@@ -789,7 +790,7 @@
       <!--Pes setting: grid          is a%ne30np4_l%ne30np4_oi%oEC60to30v3_r%r05_g%null_w%null_z%null_m%oEC60to30v3
           Pes setting: compset       is 2010_EAM%CMIP6_ELM%SPBC_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV_SIAC_SESP -->
       <pes compset=".*EAM.+ELM.+DOCN" pesize="any">
-        <comment> -compset A_WCYCL* -res ne30pg2_oECv3 without MPASO on 4 nodes, 128x1 c8 </comment>
+        <comment> -compset A_WCYCL* -res ne30pg2_oECv3 without MPASO on 4 nodes, 128x1 </comment>
         <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>512</ntasks_atm>
@@ -797,11 +798,8 @@
           <ntasks_rof>512</ntasks_rof>
           <ntasks_ice>512</ntasks_ice>
           <ntasks_ocn>512</ntasks_ocn>
-          <ntasks_cpl>64</ntasks_cpl>
+          <ntasks_cpl>512</ntasks_cpl>
         </ntasks>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
   </grid>
@@ -1092,7 +1090,7 @@
   <grid name="a%ne120np4">
     <mach name="pm-cpu|alvarez">
       <pes compset=".*EAM.+ELM.+MPASSI.+DOCN.+SGLC.+SWAV.*" pesize="any">
-        <comment>pm-cpu ne120pg2 F-compset with MPASSI on 43 nodes 128x1c8 1.3 sypd</comment>
+        <comment>pm-cpu ne120pg2 F-compset with MPASSI on 43 nodes 128x1 1.3 sypd</comment>
         <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>5504</ntasks_atm>
@@ -1102,7 +1100,7 @@
           <ntasks_ocn>5504</ntasks_ocn>
           <ntasks_glc>64</ntasks_glc>
           <ntasks_wav>64</ntasks_wav>
-          <ntasks_cpl>688</ntasks_cpl>
+          <ntasks_cpl>5504</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -1114,9 +1112,6 @@
           <nthrds_wav>1</nthrds_wav>
           <nthrds_cpl>1</nthrds_cpl>
         </nthrds>
-        <pstrid>
-          <pstrid_cpl>8</pstrid_cpl>
-        </pstrid>
       </pes>
     </mach>
     <mach name="theta">