diff --git a/mache/cime_machine_config/config_machines.xml b/mache/cime_machine_config/config_machines.xml
index ff0c583f..593acc1b 100644
--- a/mache/cime_machine_config/config_machines.xml
+++ b/mache/cime_machine_config/config_machines.xml
@@ -190,6 +190,7 @@
       <cmd_path lang="csh">module</cmd_path>
 
       <modules>
+        <command name="unload">cpe</command>
         <command name="unload">cray-hdf5-parallel</command>
         <command name="unload">cray-netcdf-hdf5parallel</command>
         <command name="unload">cray-parallel-netcdf</command>
@@ -218,8 +219,8 @@
 
       <modules compiler="gnu">
         <command name="load">PrgEnv-gnu/8.5.0</command>
-        <command name="load">gcc/12.2.0</command>
-        <command name="load">cray-libsci/23.02.1.1</command>
+        <command name="load">gcc-native/12.3</command>
+        <command name="load">cray-libsci/23.12.5</command>
       </modules>
 
       <modules compiler="intel">
@@ -229,35 +230,23 @@
 
       <modules compiler="nvidia">
         <command name="load">PrgEnv-nvidia</command>
-        <command name="load">nvidia/22.7</command>
-        <command name="load">cray-libsci/23.02.1.1</command>
+        <command name="load">nvidia/24.5</command>
+        <command name="load">cray-libsci/23.12.5</command>
       </modules>
 
       <modules compiler="amdclang">
         <command name="load">PrgEnv-aocc</command>
-        <command name="load">aocc/4.0.0</command>
-        <command name="load">cray-libsci/23.02.1.1</command>
+        <command name="load">aocc/4.1.0</command>
+        <command name="load">cray-libsci/23.12.5</command>
       </modules>
 
-      <modules compiler="intel">
+      <modules>
         <command name="load">craype-accel-host</command>
         <command name="load">craype/2.7.30</command>
         <command name="load">cray-mpich/8.1.28</command>
         <command name="load">cray-hdf5-parallel/1.12.2.9</command>
         <command name="load">cray-netcdf-hdf5parallel/4.9.0.9</command>
         <command name="load">cray-parallel-netcdf/1.12.3.9</command>
-      </modules>
-
-      <modules compiler="!intel">
-        <command name="load">craype-accel-host</command>
-        <command name="load">craype/2.7.20</command>
-        <command name="load">cray-mpich/8.1.25</command>
-        <command name="load">cray-hdf5-parallel/1.12.2.3</command>
-        <command name="load">cray-netcdf-hdf5parallel/4.9.0.3</command>
-        <command name="load">cray-parallel-netcdf/1.12.3.3</command>
-      </modules>
-
-      <modules>
         <command name="load">cmake/3.24.3</command>
         <command name="load">evp-patch</command>
       </modules>
@@ -367,6 +356,7 @@
       <cmd_path lang="csh">module</cmd_path>
 
       <modules>
+        <command name="unload">cpe</command>
         <command name="unload">cray-hdf5-parallel</command>
         <command name="unload">cray-netcdf-hdf5parallel</command>
         <command name="unload">cray-parallel-netcdf</command>
@@ -377,12 +367,14 @@
         <command name="unload">PrgEnv-nvidia</command>
         <command name="unload">PrgEnv-cray</command>
         <command name="unload">PrgEnv-aocc</command>
+        <command name="unload">gcc-native</command>
         <command name="unload">intel</command>
         <command name="unload">intel-oneapi</command>
         <command name="unload">nvidia</command>
         <command name="unload">aocc</command>
         <command name="unload">cudatoolkit</command>
         <command name="unload">climate-utils</command>
+        <command name="unload">cray-libsci</command>
         <command name="unload">matlab</command>
         <command name="unload">craype-accel-nvidia80</command>
         <command name="unload">craype-accel-host</command>
@@ -392,24 +384,24 @@
       </modules>
 
       <modules compiler="gnu.*">
-        <command name="load">PrgEnv-gnu/8.3.3</command>
-        <command name="load">gcc/11.2.0</command>
+        <command name="load">PrgEnv-gnu/8.5.0</command>
+        <command name="load">gcc-native/12.3</command>
       </modules>
 
       <modules compiler="nvidia.*">
         <command name="load">PrgEnv-nvidia</command>
-        <command name="load">nvidia/22.7</command>
+        <command name="load">nvidia/24.5</command>
       </modules>
 
       <modules compiler="gnugpu">
-        <command name="load">cudatoolkit/11.7</command>
+        <command name="load">cudatoolkit/12.2</command>
         <command name="load">craype-accel-nvidia80</command>
       </modules>
 
       <modules compiler="nvidiagpu">
-        <command name="load">cudatoolkit/11.7</command>
+        <command name="load">cudatoolkit/12.2</command>
         <command name="load">craype-accel-nvidia80</command>
-        <command name="load">gcc-mixed/11.2.0</command>
+        <command name="load">gcc-native-mixed/12.3</command>
       </modules>
 
       <modules compiler="gnu">
@@ -421,12 +413,12 @@
       </modules>
 
       <modules>
-        <command name="load">cray-libsci/23.02.1.1</command>
-        <command name="load">craype/2.7.20</command>
-        <command name="load">cray-mpich/8.1.25</command>
-        <command name="load">cray-hdf5-parallel/1.12.2.3</command>
-        <command name="load">cray-netcdf-hdf5parallel/4.9.0.3</command>
-        <command name="load">cray-parallel-netcdf/1.12.3.3</command>
+        <command name="load">cray-libsci/23.12.5</command>
+        <command name="load">craype/2.7.30</command>
+        <command name="load">cray-mpich/8.1.28</command>
+        <command name="load">cray-hdf5-parallel/1.12.2.9</command>
+        <command name="load">cray-netcdf-hdf5parallel/4.9.0.9</command>
+        <command name="load">cray-parallel-netcdf/1.12.3.9</command>
         <command name="load">cmake/3.24.3</command>
       </modules>
     </module_system>
@@ -434,6 +426,7 @@
     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
     <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
     <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
+    <TEST_MEMLEAK_TOLERANCE>0.20</TEST_MEMLEAK_TOLERANCE>
 
     <environment_variables>
       <env name="MPICH_ENV_DISPLAY">1</env>
@@ -454,6 +447,9 @@
     <environment_variables compiler="nvidiagpu">
       <env name="MPICH_GPU_SUPPORT_ENABLED">1</env>
     </environment_variables>
+    <environment_variables compiler="gnugpu">
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi}</env>
+    </environment_variables>
     <environment_variables compiler="gnu.*" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi}</env>
     </environment_variables>
@@ -588,10 +584,9 @@
       <env name="PERL5LIB">/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch</env>
       <env name="FI_CXI_RX_MATCH_MODE">software</env>
       <env name="MPICH_COLL_SYNC">MPI_Bcast</env>
-      <env name="Albany_ROOT">$SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi}</env>
-      <env name="Trilinos_ROOT">$SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi}</env>
       <env name="NETCDF_PATH">$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}</env>
       <env name="PNETCDF_PATH">$ENV{CRAY_PARALLEL_NETCDF_PREFIX}</env>
+      <env name="GATOR_INITIAL_MB">4000MB</env>
     </environment_variables>
     <environment_variables compiler="intel" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi}</env>
@@ -599,6 +594,8 @@
     <environment_variables compiler="gnu" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi}</env>
       <env name="BLA_VENDOR">Generic</env>
+      <env name="Albany_ROOT">$SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi}</env>
+      <env name="Trilinos_ROOT">$SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi}</env>
     </environment_variables>
     <environment_variables compiler="nvidia" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi}</env>
@@ -614,6 +611,13 @@
     <environment_variables compiler="amdclang" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi}</env>
     </environment_variables>
+    <environment_variables compiler="intel">
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi}</env>
+    </environment_variables>
+    <environment_variables compiler="gnu">
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi}</env>
+    </environment_variables>
+
     <resource_limits>
       <resource name="RLIMIT_STACK">-1</resource>
     </resource_limits>
@@ -680,12 +684,14 @@
         <command name="unload">PrgEnv-nvidia</command>
         <command name="unload">PrgEnv-cray</command>
         <command name="unload">PrgEnv-aocc</command>
+        <command name="unload">gcc-native</command>
         <command name="unload">intel</command>
         <command name="unload">intel-oneapi</command>
         <command name="unload">nvidia</command>
         <command name="unload">aocc</command>
         <command name="unload">cudatoolkit</command>
         <command name="unload">climate-utils</command>
+        <command name="unload">cray-libsci</command>
         <command name="unload">matlab</command>
         <command name="unload">craype-accel-nvidia80</command>
         <command name="unload">craype-accel-host</command>
@@ -695,26 +701,24 @@
       </modules>
 
       <modules compiler="gnu.*">
-        <command name="load">PrgEnv-gnu/8.3.3</command>
-        <command name="load">gcc/11.2.0</command>
-        <!--command name="load">PrgEnv-gnu/8.5.0</command>
-        <command name="load">gcc-native/12.3</command-->
+        <command name="load">PrgEnv-gnu/8.5.0</command>
+        <command name="load">gcc-native/12.3</command>
       </modules>
 
       <modules compiler="nvidia.*">
         <command name="load">PrgEnv-nvidia</command>
-        <command name="load">nvidia/23.9</command>
+        <command name="load">nvidia/24.5</command>
       </modules>
 
       <modules compiler="gnugpu">
-        <command name="load">cudatoolkit/11.7</command>
-        <!--command name="load">cudatoolkit/12.2</command-->
+        <command name="load">cudatoolkit/12.2</command>
         <command name="load">craype-accel-nvidia80</command>
       </modules>
 
       <modules compiler="nvidiagpu">
-        <command name="load">cudatoolkit/11.7</command>
+        <command name="load">cudatoolkit/12.2</command>
         <command name="load">craype-accel-nvidia80</command>
+        <command name="load">gcc-native-mixed/12.3</command>
       </modules>
 
       <modules compiler="gnu">
@@ -726,26 +730,20 @@
       </modules>
 
       <modules>
-        <command name="load">cray-libsci/23.02.1.1</command>
-        <command name="load">craype/2.7.20</command>
-        <command name="load">cray-mpich/8.1.25</command>
-        <command name="load">cray-hdf5-parallel/1.12.2.3</command>
-        <command name="load">cray-netcdf-hdf5parallel/4.9.0.3</command>
-        <command name="load">cray-parallel-netcdf/1.12.3.3</command>
-        <!--command name="load">cray-libsci/23.12.5</command>
+        <command name="load">cray-libsci/23.12.5</command>
         <command name="load">craype/2.7.30</command>
         <command name="load">cray-mpich/8.1.28</command>
         <command name="load">cray-hdf5-parallel/1.12.2.9</command>
         <command name="load">cray-netcdf-hdf5parallel/4.9.0.9</command>
-        <command name="load">cray-parallel-netcdf/1.12.3.9</command-->
+        <command name="load">cray-parallel-netcdf/1.12.3.9</command>
         <command name="load">cmake/3.24.3</command>
-        <command name="load">evp-patch</command>
       </modules>
     </module_system>
 
     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
     <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
     <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
+    <TEST_MEMLEAK_TOLERANCE>0.20</TEST_MEMLEAK_TOLERANCE>
 
     <environment_variables>
       <env name="MPICH_ENV_DISPLAY">1</env>
@@ -766,6 +764,9 @@
     <environment_variables compiler="nvidiagpu">
       <env name="MPICH_GPU_SUPPORT_ENABLED">1</env>
     </environment_variables>
+    <environment_variables compiler="gnugpu">
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi}</env>
+    </environment_variables>
     <environment_variables compiler="gnu.*" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi}</env>
     </environment_variables>
@@ -823,6 +824,7 @@
       <cmd_path lang="csh">module</cmd_path>
 
       <modules>
+        <command name="unload">cpe</command>
         <command name="unload">cray-hdf5-parallel</command>
         <command name="unload">cray-netcdf-hdf5parallel</command>
         <command name="unload">cray-parallel-netcdf</command>
@@ -850,36 +852,35 @@
       </modules>
 
       <modules compiler="gnu">
-        <command name="load">PrgEnv-gnu</command>
-        <command name="load">gcc-native</command>
-        <command name="load">cray-libsci</command>
+        <command name="load">PrgEnv-gnu/8.5.0</command>
+        <command name="load">gcc-native/13.2</command>
+        <command name="load">cray-libsci/24.03.0</command>
       </modules>
 
       <modules compiler="intel">
-        <command name="load">PrgEnv-intel</command>
-        <command name="load">intel</command>
+        <command name="load">PrgEnv-intel/8.5.0</command>
+        <command name="load">intel/2024.1.0</command>
       </modules>
 
       <modules compiler="nvidia">
         <command name="load">PrgEnv-nvidia</command>
         <command name="load">nvidia/24.5</command>
-        <command name="load">cray-libsci</command>
+        <command name="load">cray-libsci/24.03.0</command>
       </modules>
 
       <modules compiler="amdclang">
         <command name="load">PrgEnv-aocc</command>
-        <command name="load">aocc/4.0.1</command>
-        <command name="load">cray-libsci</command>
+        <command name="load">aocc/4.1.0</command>
+        <command name="load">cray-libsci/24.03.0</command>
       </modules>
 
       <modules>
         <command name="load">craype-accel-host</command>
-        <command name="load">cray-libsci</command>
-        <command name="load">craype/2.7.30</command>
-        <command name="load">cray-mpich/8.1.28</command>
-        <command name="load">cray-hdf5-parallel/1.12.2.9</command>
-        <command name="load">cray-netcdf-hdf5parallel/4.9.0.9</command>
-        <command name="load">cray-parallel-netcdf/1.12.3.9</command>
+        <command name="load">craype/2.7.31.11</command>
+        <command name="load">cray-mpich/8.1.29</command>
+        <command name="load">cray-hdf5-parallel/1.12.2.11</command>
+        <command name="load">cray-netcdf-hdf5parallel/4.9.0.11</command>
+        <command name="load">cray-parallel-netcdf/1.12.3.11</command>
         <command name="load">cmake/3.24.3</command>
       </modules>
     </module_system>
@@ -887,6 +888,7 @@
     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
     <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
     <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
+    <TEST_MEMLEAK_TOLERANCE>0.20</TEST_MEMLEAK_TOLERANCE>
 
     <environment_variables>
       <env name="MPICH_ENV_DISPLAY">1</env>
@@ -901,12 +903,44 @@
       <env name="MPICH_COLL_SYNC">MPI_Bcast</env>
       <env name="NETCDF_PATH">$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}</env>
       <env name="PNETCDF_PATH">$ENV{CRAY_PARALLEL_NETCDF_PREFIX}</env>
+      <env name="GATOR_INITIAL_MB">4000MB</env>
+    </environment_variables>
+    <environment_variables compiler="intel" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi}</env>
+    </environment_variables>
+    <environment_variables compiler="gnu" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi}</env>
+      <env name="BLA_VENDOR">Generic</env>
+      <env name="Albany_ROOT">$SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi}</env>
+      <env name="Trilinos_ROOT">$SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi}</env>
+    </environment_variables>
+    <environment_variables compiler="nvidia" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi}</env>
+    </environment_variables>
+    <environment_variables compiler="nvidia">
+      <env name="BLAS_ROOT">$SHELL{if [ -z "$BLAS_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$BLAS_ROOT"; fi}</env>
+      <env name="LAPACK_ROOT">$SHELL{if [ -z "$LAPACK_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$LAPACK_ROOT"; fi}</env>
+      <env name="BLA_VENDOR">NVHPC</env>
+    </environment_variables>
+    <environment_variables compiler="intel">
+      <env name="BLA_VENDOR">Intel10_64_dyn</env>
+    </environment_variables>
+    <environment_variables compiler="amdclang" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi}</env>
     </environment_variables>
+    <environment_variables compiler="intel">
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi}</env>
+    </environment_variables>
+    <environment_variables compiler="gnu">
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi}</env>
+    </environment_variables>
+
     <resource_limits>
       <resource name="RLIMIT_STACK">-1</resource>
     </resource_limits>
   </machine>
 
+
   <machine MACH="spock">
     <DESC>Spock. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL.</DESC>
     <NODENAME_REGEX>.*spock.*</NODENAME_REGEX>
@@ -1115,306 +1149,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
 -->
   </machine>
 
-  <machine MACH="crusher">
-    <DESC>Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs</DESC>
-    <NODENAME_REGEX>.*crusher.*</NODENAME_REGEX>
-    <OS>Linux</OS>
-    <COMPILERS>crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu</COMPILERS>
-    <MPILIBS>mpich</MPILIBS>
-    <PROJECT>cli115</PROJECT>
-    <SAVE_TIMING_DIR>/lustre/orion/cli115/world-shared/crusher</SAVE_TIMING_DIR>
-    <SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
-    <CIME_OUTPUT_ROOT>/lustre/orion/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch/crusher</CIME_OUTPUT_ROOT>
-    <DIN_LOC_ROOT>/lustre/orion/cli115/world-shared/e3sm/inputdata</DIN_LOC_ROOT>
-    <DIN_LOC_ROOT_CLMFORC>/lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
-    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
-    <BASELINE_ROOT>/lustre/orion/cli115/world-shared/e3sm/baselines/crusher/$COMPILER</BASELINE_ROOT>
-    <CCSM_CPRNC>/lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc</CCSM_CPRNC>
-    <GMAKE_J>8</GMAKE_J>
-    <NTEST_PARALLEL_JOBS>1</NTEST_PARALLEL_JOBS>
-    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
-    <SUPPORTED_BY>e3sm</SUPPORTED_BY>
-    <MAX_TASKS_PER_NODE>56</MAX_TASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE>56</MAX_MPITASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE compiler="gnugpu">8</MAX_MPITASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE compiler="crayclanggpu">8</MAX_MPITASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE compiler="amdclanggpu">8</MAX_MPITASKS_PER_NODE>
-    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
-    <mpirun mpilib="default">
-      <executable>srun</executable>
-      <arguments>
-        <arg name="num_tasks"> -l -K -n {{ total_tasks }} -N {{ num_nodes }} </arg>
-        <arg name="thread_count">-c $ENV{OMP_NUM_THREADS}</arg>
-        <arg name="ntasks_per_gpu">$ENV{NTASKS_PER_GPU}</arg>
-        <arg name="gpu_bind">$ENV{GPU_BIND_ARGS}</arg>
-      </arguments>
-    </mpirun>
-    <module_system type="module" allow_error="true">
-      <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
-      <init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
-      <init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
-      <init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
-      <cmd_path lang="perl">/usr/share/lmod/lmod/libexec/lmod perl</cmd_path>
-      <cmd_path lang="sh">module</cmd_path>
-      <cmd_path lang="csh">module</cmd_path>
-      <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
-      <modules compiler="crayclang.*">
-        <command name="reset"></command>
-        <command name="switch">PrgEnv-cray PrgEnv-cray/8.3.3</command>
-        <command name="switch">cce cce/15.0.1</command>
-        <!-- craype module to address tcmalloc runtime errors at startup -->
-        <!-- tcmalloc.cc:647 Attempt to free invalid pointer -->
-        <command name="switch">craype craype/2.7.20</command>
-      </modules>
-      <modules compiler="crayclanggpu">
-        <command name="load">craype-accel-amd-gfx90a</command>
-        <command name="load">rocm/5.4.0</command>
-      </modules>
-      <modules compiler="amdclang.*">
-        <command name="reset"></command>
-        <command name="switch">PrgEnv-cray PrgEnv-amd/8.3.3</command>
-        <command name="switch">amd amd/5.4.0</command>
-      </modules>
-      <modules compiler="amdclanggpu">
-        <command name="load">craype-accel-amd-gfx90a</command>
-      </modules>
-      <modules compiler="gnu.*">
-        <command name="reset"></command>
-        <command name="switch">PrgEnv-cray PrgEnv-gnu/8.3.3</command>
-        <command name="switch">gcc gcc/11.2.0</command>
-      </modules>
-      <modules compiler="gnugpu">
-        <command name="load">craype-accel-amd-gfx90a</command>
-        <command name="load">rocm/5.4.0</command>
-      </modules>
-      <modules>
-        <command name="load">cray-python/3.9.13.1</command>
-        <command name="load">subversion/1.14.1</command>
-        <command name="load">git/2.36.1</command>
-        <command name="load">cmake/3.21.3</command>
-        <command name="load">zlib/1.2.11</command>
-        <command name="load">cray-hdf5-parallel/1.12.2.1</command>
-        <command name="load">cray-netcdf-hdf5parallel/4.9.0.1</command>
-        <command name="load">cray-parallel-netcdf/1.12.3.1</command>
-      </modules>
-    </module_system>
-    <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
-    <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
-    <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
-    <TEST_MEMLEAK_TOLERANCE>0.25</TEST_MEMLEAK_TOLERANCE>
-    <MAX_GB_OLD_TEST_DATA>0</MAX_GB_OLD_TEST_DATA>
-    <environment_variables>
-      <env name="NETCDF_PATH">$ENV{NETCDF_DIR}</env>
-      <env name="PNETCDF_PATH">$ENV{PNETCDF_DIR}</env>
-      <env name="NTASKS_PER_GPU"> </env>
-      <env name="GPU_BIND_ARGS"> </env>
-    </environment_variables>
-    <environment_variables compiler="amdclang">
-      <env name="LD_LIBRARY_PATH">$ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH}</env>
-    </environment_variables>
-    <environment_variables compiler=".*gpu.*">
-      <env name="NTASKS_PER_GPU">--ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc}</env>
-      <env name="GPU_BIND_ARGS">--gpu-bind=closest</env>
-      <env name="PNETCDF_HINTS">romio_cb_read=disable</env>
-      <env name="MPICH_GPU_SUPPORT_ENABLED">0</env>
-    </environment_variables>
-    <environment_variables compiler=".*gpu.*" DEBUG="TRUE">
-      <env name="AMD_LOG_LEVEL">10</env>
-      <env name="CRAY_ACC_DEBUG">3</env>
-    </environment_variables>
-    <environment_variables BUILD_THREADED="TRUE">
-      <env name="OMP_STACKSIZE">128M</env>
-      <env name="OMP_PROC_BIND">spread</env>
-      <env name="OMP_PLACES">threads</env>
-    </environment_variables>
-  </machine>
-
-  <machine MACH="crusher-scream-cpu">
-    <DESC>Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs</DESC>
-    <NODENAME_REGEX>.*crusher.*</NODENAME_REGEX>
-    <OS>Linux</OS>
-    <COMPILERS>crayclang-scream</COMPILERS>
-    <MPILIBS>mpich</MPILIBS>
-    <PROJECT>CLI115</PROJECT>
-    <CIME_OUTPUT_ROOT>/lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher</CIME_OUTPUT_ROOT>
-    <DIN_LOC_ROOT>/lustre/orion/cli115/world-shared/e3sm/inputdata</DIN_LOC_ROOT>
-    <DIN_LOC_ROOT_CLMFORC>/lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
-    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
-    <BASELINE_ROOT>/lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER</BASELINE_ROOT>
-    <CCSM_CPRNC>/lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc</CCSM_CPRNC>
-    <GMAKE_J>8</GMAKE_J>
-    <NTEST_PARALLEL_JOBS>1</NTEST_PARALLEL_JOBS>
-    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
-    <SUPPORTED_BY>e3sm</SUPPORTED_BY>
-    <MAX_TASKS_PER_NODE>56</MAX_TASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE>56</MAX_MPITASKS_PER_NODE>
-    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
-
-    <mpirun mpilib="default">
-      <executable>srun</executable>
-      <arguments>
-        <arg name="num_tasks"> -l -K -n {{ total_tasks }} -N {{ num_nodes }} </arg>
-        <!-- <arg name="binding">cpu_bind=cores</arg>  -->
-        <!-- <arg name="binding">threads-per-core=1 -c 2</arg> -->
-        <arg name="binding">--threads-per-core=1</arg>
-        <arg name="thread_count">-c $ENV{OMP_NUM_THREADS}</arg>
-        <arg name="placement">-m *:block</arg>
-        <!--<arg name="placement">-m plane={{ tasks_per_node }}</arg>-->
-      </arguments>
-    </mpirun>
-
-    <module_system type="module" allow_error="true">
-      <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
-      <init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
-      <init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
-      <init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
-      <cmd_path lang="perl">/usr/share/lmod/lmod/libexec/lmod perl</cmd_path>
-      <cmd_path lang="sh">module</cmd_path>
-      <cmd_path lang="csh">module</cmd_path>
-      <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
-
-      <modules compiler="crayclang-scream">
-        <command name="reset"></command>
-        <command name="switch">PrgEnv-cray PrgEnv-cray/8.3.3</command>
-        <command name="switch">cce cce/14.0.0</command>
-      </modules>
-
-      <modules compiler="amdclang">
-        <command name="reset"></command>
-        <command name="switch">PrgEnv-cray PrgEnv-amd/8.3.3</command>
-        <command name="switch">amd amd/5.1.0</command>
-      </modules>
-
-      <modules compiler="gnu">
-        <command name="reset"></command>
-        <command name="switch">PrgEnv-cray PrgEnv-gnu/8.3.3</command>
-      </modules>
-      <modules>
-        <command name="load">cray-mpich/8.1.12</command>
-        <command name="load">cray-python/3.9.4.2</command>
-        <command name="load">subversion/1.14.0</command>
-        <command name="load">git/2.31.1</command>
-        <command name="load">cmake/3.21.3</command>
-        <command name="load">zlib/1.2.11</command>
-        <command name="load">cray-libsci/21.08.1.2</command>
-        <command name="load">cray-hdf5-parallel/1.12.1.1</command>
-        <command name="load">cray-netcdf-hdf5parallel/4.8.1.1</command>
-        <command name="load">cray-parallel-netcdf/1.12.1.7</command>
-      </modules>
-
-    </module_system>
-    <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
-    <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
-    <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
-    <environment_variables>
-      <env name="NETCDF_PATH">$ENV{NETCDF_DIR}</env>
-      <env name="PNETCDF_PATH">$ENV{PNETCDF_DIR}</env>
-    </environment_variables>
-
-    <environment_variables compiler="amdclang">
-      <env name="LD_LIBRARY_PATH">$ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH}</env>
-    </environment_variables>
-
-    <environment_variables>
-      <env name="MPICH_GPU_SUPPORT_ENABLED">0</env>
-    </environment_variables>
-
-    <environment_variables BUILD_THREADED="TRUE">
-      <env name="OMP_STACKSIZE">128M</env>
-      <env name="OMP_PROC_BIND">spread</env>
-      <env name="OMP_PLACES">threads</env>
-    </environment_variables>
-  </machine>
-
-  <machine MACH="crusher-scream-gpu">
-    <DESC>Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs</DESC>
-    <NODENAME_REGEX>.*crusher.*</NODENAME_REGEX>
-    <OS>Linux</OS>
-    <COMPILERS>crayclang-scream</COMPILERS>
-    <MPILIBS>mpich</MPILIBS>
-    <PROJECT>CLI115</PROJECT>
-    <CIME_OUTPUT_ROOT>/lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher</CIME_OUTPUT_ROOT>
-    <DIN_LOC_ROOT>/lustre/orion/cli115/world-shared/e3sm/inputdata</DIN_LOC_ROOT>
-    <DIN_LOC_ROOT_CLMFORC>/lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
-    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
-    <BASELINE_ROOT>/lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER</BASELINE_ROOT>
-    <CCSM_CPRNC>/lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc</CCSM_CPRNC>
-    <GMAKE_J>8</GMAKE_J>
-    <NTEST_PARALLEL_JOBS>1</NTEST_PARALLEL_JOBS>
-    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
-    <SUPPORTED_BY>e3sm</SUPPORTED_BY>
-    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE>8</MAX_MPITASKS_PER_NODE>
-    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
-
-    <mpirun mpilib="default">
-      <executable>srun</executable>
-      <arguments>
-        <arg name="num_tasks"> -l -K -n {{ total_tasks }} -N {{ num_nodes }} </arg>
-        <!-- <arg name="binding">cpu_bind=cores</arg>  -->
-        <!-- <arg name="binding">threads-per-core=1 -c 2</arg> -->
-        <arg name="binding">--gpus-per-node=8 --gpu-bind=closest</arg>
-        <arg name="thread_count">-c $ENV{OMP_NUM_THREADS}</arg>
-        <!-- <arg name="placement">-m *:block</arg> -->
-        <!--<arg name="placement">-m plane={{ tasks_per_node }}</arg>-->
-      </arguments>
-    </mpirun>
-
-    <module_system type="module" allow_error="true">
-      <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
-      <init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
-      <init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
-      <init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
-      <cmd_path lang="perl">/usr/share/lmod/lmod/libexec/lmod perl</cmd_path>
-      <cmd_path lang="sh">module</cmd_path>
-      <cmd_path lang="csh">module</cmd_path>
-      <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
-      <modules compiler="crayclang-scream">
-        <command name="reset"></command>
-        <command name="load">PrgEnv-cray</command>
-        <!-- See SCREAM issue #2080. -->
-        <command name="load">craype-accel-amd-gfx90a</command>
-        <command name="load">rocm/5.1.0</command>
-        <!-- Work around tcmalloc crash. -->
-        <command name="load">cce/14.0.3</command>
-      </modules>
-      <modules>
-        <command name="load">cray-python/3.9.4.2</command>
-        <command name="load">subversion/1.14.0</command>
-        <command name="load">git/2.31.1</command>
-        <command name="load">cmake/3.21.3</command>
-        <command name="load">zlib/1.2.11</command>
-        <command name="load">cray-hdf5-parallel/1.12.2.1</command>
-        <command name="load">cray-netcdf-hdf5parallel/4.9.0.1</command>
-        <command name="load">cray-parallel-netcdf/1.12.3.1</command>
-      </modules>
-
-    </module_system>
-    <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
-    <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
-    <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
-    <MAX_GB_OLD_TEST_DATA>0</MAX_GB_OLD_TEST_DATA>
-    <environment_variables>
-      <env name="NETCDF_PATH">$ENV{NETCDF_DIR}</env>
-      <env name="PNETCDF_PATH">$ENV{PNETCDF_DIR}</env>
-      <env name="MPIR_CVAR_GPU_EAGER_DEVICE_MEM">0</env>
-      <!-- See SCREAM issue #2080. -->
-      <env name="MPICH_GPU_SUPPORT_ENABLED">1</env>
-      <env name="PNETCDF_HINTS">romio_cb_read=disable</env>
-    </environment_variables>
-
-    <environment_variables BUILD_THREADED="TRUE">
-      <env name="OMP_STACKSIZE">128M</env>
-      <env name="OMP_PROC_BIND">spread</env>
-      <env name="OMP_PLACES">threads</env>
-    </environment_variables>
-  </machine>
-
   <machine MACH="frontier-scream-gpu">
     <DESC>Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs.</DESC>
     <NODENAME_REGEX>.*frontier.*</NODENAME_REGEX>
     <OS>CNL</OS>
-    <COMPILERS>crayclang-scream</COMPILERS>
+    <COMPILERS>craygnuamdgpu,crayclang-scream</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>cli115</PROJECT>
     <SAVE_TIMING_DIR>/lustre/orion/proj-shared/cli115</SAVE_TIMING_DIR>
@@ -1451,17 +1190,34 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
       <cmd_path lang="sh">module</cmd_path>
       <cmd_path lang="csh">module</cmd_path>
       <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
+      <modules compiler="craygnuamdgpu">
+        <command name="reset"></command>
+        <!-- PrgEnv-gnu before cpe, or cray-mpich module doesn't update correctly -->
+        <command name="load">PrgEnv-gnu</command>
+        <command name="load">cpe/24.07</command>
+        <command name="load">libfabric/1.15.2.0</command>
+        <command name="load">craype-accel-amd-gfx90a</command>
+        <command name="load">rocm/6.2.0</command>
+        <command name="load">libunwind</command>
+        <command name="load">cray-python</command>
+        <command name="load">subversion</command>
+        <command name="load">git</command>
+        <command name="load">cmake</command>
+        <command name="load">cray-hdf5-parallel</command>
+        <command name="load">cray-netcdf-hdf5parallel</command>
+        <command name="load">cray-parallel-netcdf</command>
+        <command name="unload">darshan-runtime</command>
+      </modules>
       <modules compiler="crayclang-scream">
         <command name="reset"></command>
-        <command name="load">PrgEnv-cray</command>
+        <command name="load">cpe/22.12</command>
         <command name="load">craype-accel-amd-gfx90a</command>
         <command name="load">rocm/5.4.0</command>
         <command name="load">libunwind/1.6.2</command>
-      </modules>
-      <modules>
         <command name="load">cce/15.0.1</command>
-        <command name="switch">craype craype/2.7.20</command>
-        <command name="switch">cray-mpich cray-mpich/8.1.26</command>
+        <command name="load">libfabric/1.15.2.0</command>
+        <command name="load">craype/2.7.20</command>
+        <command name="load">cray-mpich/8.1.26</command>
         <command name="load">cray-python/3.9.13.1</command>
         <command name="load">subversion/1.14.1</command>
         <command name="load">git/2.36.1</command>
@@ -1495,9 +1251,13 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
       <env name="OMP_PLACES">threads</env>
     </environment_variables>
 
+    <environment_variables compiler="craygnuamdgpu" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.10/install/craygnuamdgppu/cpe-24.07/libfabric-1.15.2.0; else echo "$ADIOS2_ROOT"; fi}</env>
+    </environment_variables>
     <environment_variables compiler="crayclang-scream" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi}</env>
     </environment_variables>
+
   </machine>
 
   <!-- Skylake nodes of Stampede2 at TACC -->
@@ -3044,6 +2804,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
       <env name="PNETCDF_PATH">/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6</env>
       <env name="PATH">/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH}</env>
       <env name="LD_LIBRARY_PATH">$SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi}</env>
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/improv/gnu; else echo "$MOAB_ROOT"; fi}</env>
       <env name="OMPI_MCA_sharedfp">^lockedfile</env>
     </environment_variables>
     <environment_variables BUILD_THREADED="TRUE">
@@ -3065,9 +2826,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
     <DIN_LOC_ROOT_CLMFORC>/usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
     <DOUT_S_ROOT>/p/lustre2/$USER/archive/$CASE</DOUT_S_ROOT>
     <BASELINE_ROOT>/p/lustre2/$USER/ccsm_baselines/$COMPILER</BASELINE_ROOT>
-    <CCSM_CPRNC>/usr/workspace/e3sm/tools/cprnc</CCSM_CPRNC>
+    <CCSM_CPRNC>/usr/workspace/e3sm/apps/cprnc</CCSM_CPRNC>
     <GMAKE_J>8</GMAKE_J>
-    <BATCH_SYSTEM>lc_slurm</BATCH_SYSTEM>
+    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
     <SUPPORTED_BY>boutte3 -at- llnl.gov</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>56</MAX_TASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>56</MAX_MPITASKS_PER_NODE>
@@ -3076,8 +2837,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
     </mpirun>
     <mpirun mpilib="default">
       <executable>srun</executable>
+      <arguments>
+        <arg name="pmi_layer">--mpi=pmi2</arg>
+        <arg name="export">--export=ALL</arg>
+        <arg name="num_tasks">-n {{ total_tasks }} -N {{ num_nodes }}</arg>
+        <arg name="thread_count">-c 1</arg>
+        <arg name="binding">--cpu_bind=cores</arg>
+        <arg name="placement">-m plane={{ tasks_per_node }}</arg>
+      </arguments>
     </mpirun>
-    <module_system type="module">
+    <module_system type="module" allow_error="true">
       <init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
       <init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
       <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
@@ -3089,24 +2858,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
       <modules compiler="intel">
         <command name="load">python/3.9.12</command>
         <command name="load">git</command>
+        <command name="load">subversion</command>
+        <command name="load">cmake/3.19.2</command>
         <command name="load">mkl/2022.1.0</command>
         <command name="load">intel-classic/2021.6.0-magic</command>
-        <command name="load">mvapich2/2.3.7</command>
-        <command name="load">cmake/3.19.2</command>
-        <command name="use --append">/usr/workspace/e3sm/install/quartz/modulefiles</command>
-        <command name="load">hdf5/1.12.2</command>
-        <command name="load">netcdf-c/4.9.0</command>
-        <command name="load">netcdf-fortran/4.6.0</command>
-        <command name="load">parallel-netcdf/1.12.3</command>
-        <command name="load">screamML-venv/0.0.1</command>
-        <command name="load">subversion</command>
+        <command name="use --append">/usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core</command>
+        <command name="load">mvapich2/2.3.7-ll7cmqm</command>
+        <command name="load">hdf5/1.10.7-ewjpbjd</command>
+        <command name="load">netcdf-c/4.4.1.1-vaxofek</command>
+        <command name="load">netcdf-fortran/4.4.4-3pzbx2u</command>
+        <command name="load">parallel-netcdf/1.11.0-tzgdala</command>
       </modules>
     </module_system>
     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
     <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
     <environment_variables compiler="intel">
-      <env name="NETCDF_PATH">/usr/workspace/e3sm/install/quartz/netcdf-fortran/</env>
-      <env name="PNETCDF_PATH">/usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0</env>
+      <env name="OMP_STACKSIZE">128M</env>
+      <env name="HDF5_USE_FILE_LOCKING">FALSE</env>
+      <env name="HDF5_ROOT">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg</env>
+      <env name="NETCDF_C_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn</env>
+      <env name="NETCDF_FORTRAN_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2</env>
+      <env name="PNETCDF_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5</env>
     </environment_variables>
   </machine>
 
@@ -3121,9 +2893,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
     <DIN_LOC_ROOT_CLMFORC>/usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
     <DOUT_S_ROOT>/p/lustre2/$USER/archive/$CASE</DOUT_S_ROOT>
     <BASELINE_ROOT>/p/lustre2/$USER/ccsm_baselines/$COMPILER</BASELINE_ROOT>
-    <CCSM_CPRNC>/usr/workspace/e3sm/tools/cprnc</CCSM_CPRNC>
+    <CCSM_CPRNC>/usr/workspace/e3sm/apps/cprnc</CCSM_CPRNC>
     <GMAKE_J>8</GMAKE_J>
-    <BATCH_SYSTEM>lc_slurm</BATCH_SYSTEM>
+    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
     <SUPPORTED_BY>boutte3 -at- llnl.gov</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>224</MAX_TASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>112</MAX_MPITASKS_PER_NODE>
@@ -3132,8 +2904,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
     </mpirun>
     <mpirun mpilib="default">
       <executable>srun</executable>
+      <arguments>
+        <arg name="pmi_layer">--mpi=pmi2</arg>
+        <arg name="export">--export=ALL</arg>
+        <arg name="num_tasks">-n {{ total_tasks }} -N {{ num_nodes }}</arg>
+        <arg name="thread_count">-c 1</arg>
+        <arg name="binding">--cpu_bind=cores</arg>
+        <arg name="placement">-m plane={{ tasks_per_node }}</arg>
+      </arguments>
     </mpirun>
-    <module_system type="module">
+    <module_system type="module" allow_error="true">
       <init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
       <init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
       <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
@@ -3145,24 +2925,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
       <modules compiler="intel">
         <command name="load">python/3.9.12</command>
         <command name="load">git</command>
+        <command name="load">subversion</command>
         <command name="load">mkl/2022.1.0</command>
         <command name="load">intel-classic/2021.6.0-magic</command>
-        <command name="load">mvapich2/2.3.7</command>
         <command name="load">cmake/3.19.2</command>
-        <command name="use --append">/usr/workspace/e3sm/install/quartz/modulefiles</command>
-        <command name="load">hdf5/1.12.2</command>
-        <command name="load">netcdf-c/4.9.0</command>
-        <command name="load">netcdf-fortran/4.6.0</command>
-        <command name="load">parallel-netcdf/1.12.3</command>
-        <command name="load">screamML-venv/0.0.1</command>
-        <command name="load">subversion</command>
+        <command name="use --append">/usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core</command>
+        <command name="load">mvapich2/2.3.7-27jao34</command>
+        <command name="load">hdf5/1.10.7-766kapa</command>
+        <command name="load">netcdf-c/4.4.1.1-2uznnlw</command>
+        <command name="load">netcdf-fortran/4.4.4-itpstyo</command>
+        <command name="load">parallel-netcdf/1.11.0-26sxm4m</command>
       </modules>
     </module_system>
     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
     <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
     <environment_variables compiler="intel">
-      <env name="NETCDF_PATH">/usr/workspace/e3sm/install/quartz/netcdf-fortran/</env>
-      <env name="PNETCDF_PATH">/usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0</env>
+      <env name="OMP_STACKSIZE">128M</env>
+      <env name="HDF5_USE_FILE_LOCKING">FALSE</env>
+      <env name="HDF5_ROOT">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv</env>
+      <env name="NETCDF_C_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib</env>
+      <env name="NETCDF_FORTRAN_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp</env>
+      <env name="PNETCDF_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck</env>
     </environment_variables>
   </machine>
 
diff --git a/mache/spack/pm-cpu_gnu_mpich.csh b/mache/spack/pm-cpu_gnu_mpich.csh
index 14e3845f..174ef24f 100644
--- a/mache/spack/pm-cpu_gnu_mpich.csh
+++ b/mache/spack/pm-cpu_gnu_mpich.csh
@@ -1,41 +1,38 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm gcc-native &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm climate-utils &> /dev/null
-module rm cray-libsci &> /dev/null
-module rm matlab &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-gnu/8.5.0
-module load gcc/12.2.0
-module load craype-accel-host
-{% if e3sm_lapack %}
-module load cray-libsci/23.02.1.1
-{% endif %}
-module load craype/2.7.20
-module rm cray-mpich &> /dev/null
-module load libfabric/1.15.2.0
-module load cray-mpich/8.1.25
+module load PrgEnv-gnu/8.5.0 \
+            gcc-native/12.3 \
+            cray-libsci/23.12.5 \
+            craype-accel-host \
+            craype/2.7.30 \
+            libfabric/1.20.1 \
+            cray-mpich/8.1.28 \
+            cmake/3.24.3
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -55,6 +52,7 @@ setenv HDF5_USE_FILE_LOCKING FALSE
 ## Not needed
 # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
 setenv FI_CXI_RX_MATCH_MODE software
+setenv FI_MR_CACHE_MONITOR kdreg2
 setenv MPICH_COLL_SYNC MPI_Bcast
 setenv GATOR_INITIAL_MB 4000MB
 setenv BLA_VENDOR Generic
diff --git a/mache/spack/pm-cpu_gnu_mpich.sh b/mache/spack/pm-cpu_gnu_mpich.sh
index 75fb3870..b824d419 100644
--- a/mache/spack/pm-cpu_gnu_mpich.sh
+++ b/mache/spack/pm-cpu_gnu_mpich.sh
@@ -1,41 +1,38 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm gcc-native &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm climate-utils &> /dev/null
-module rm cray-libsci &> /dev/null
-module rm matlab &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-gnu/8.5.0
-module load gcc/12.2.0
-module load craype-accel-host
-{% if e3sm_lapack %}
-module load cray-libsci/23.02.1.1
-{% endif %}
-module load craype/2.7.20
-module rm cray-mpich &> /dev/null
-module load libfabric/1.15.2.0
-module load cray-mpich/8.1.25
+module load PrgEnv-gnu/8.5.0 \
+            gcc-native/12.3 \
+            cray-libsci/23.12.5 \
+            craype-accel-host \
+            craype/2.7.30 \
+            libfabric/1.20.1 \
+            cray-mpich/8.1.28 \
+            cmake/3.24.3
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -55,6 +52,7 @@ export HDF5_USE_FILE_LOCKING=FALSE
 ## Not needed
 # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
 export FI_CXI_RX_MATCH_MODE=software
+export FI_MR_CACHE_MONITOR=kdreg2
 
 if [ -z "${NERSC_HOST:-}" ]; then
   # happens when building spack environment
diff --git a/mache/spack/pm-cpu_gnu_mpich.yaml b/mache/spack/pm-cpu_gnu_mpich.yaml
index 1a8b5d6d..7fd22e6f 100644
--- a/mache/spack/pm-cpu_gnu_mpich.yaml
+++ b/mache/spack/pm-cpu_gnu_mpich.yaml
@@ -2,9 +2,7 @@ spack:
   specs:
   - gcc
   - cray-mpich
-{% if e3sm_lapack %}
   - cray-libsci
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
   - hdf5
   - netcdf-c
@@ -16,12 +14,10 @@ spack:
     unify: when_possible
   packages:
     all:
-      compiler: [gcc@12.2.0]
+      compiler: [gcc@12.3]
       providers:
-        mpi: [cray-mpich@8.1.25]
-{% if e3sm_lapack %}
-        lapack: [cray-libsci@23.02.1.1]
-{% endif %}
+        mpi: [cray-mpich@8.1.28]
+        lapack: [cray-libsci@23.12.5]
     bzip2:
       externals:
       - spec: bzip2@1.0.6
@@ -101,65 +97,55 @@ spack:
       buildable: false
     gcc:
       externals:
-      - spec: gcc@12.2.0
+      - spec: gcc@12.3
         modules:
         - PrgEnv-gnu/8.5.0
-        - gcc/12.2.0
+        - gcc-native/12.3
+        - cray-libsci/23.12.5
         - craype-accel-host
-        - craype/2.7.20
-        - libfabric/1.15.2.0
+        - craype/2.7.30
+        - libfabric/1.20.1
       buildable: false
     cray-mpich:
       externals:
-      - spec: cray-mpich@8.1.25
-        prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1
-        modules:
-        - libfabric/1.15.2.0
-        - cray-mpich/8.1.25
-      buildable: false
-    libfabric:
-      externals:
-      - spec: libfabric@1.15.2.0
-        prefix: /opt/cray/libfabric/1.15.2.0
+      - spec: cray-mpich@8.1.28
         modules:
-        - libfabric/1.15.2.0
+        - libfabric/1.20.1
+        - cray-mpich/8.1.28
       buildable: false
-{% if e3sm_lapack %}
     cray-libsci:
       externals:
-      - spec: cray-libsci@23.02.1.1
-        prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64
+      - spec: cray-libsci@23.12.5
         modules:
-        - cray-libsci/23.02.1.1
+        - cray-libsci/23.12.5
       buildable: false
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
     hdf5:
       externals:
-      - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1
+      - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3
       buildable: false
     parallel-netcdf:
       externals:
-      - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared
-        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/GNU/9.1/
+      - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3
       buildable: false
     netcdf-c:
       externals:
-      - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
+      - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
       buildable: false
     netcdf-fortran:
       externals:
       - spec: netcdf-fortran@4.5.3
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
       buildable: false
 {% endif %}
   config:
     install_missing_compilers: false
   compilers:
   - compiler:
-      spec: gcc@12.2.0
+      spec: gcc@12.3
       paths:
         cc: cc
         cxx: CC
@@ -170,10 +156,11 @@ spack:
       target: x86_64
       modules:
       - PrgEnv-gnu/8.5.0
-      - gcc/12.2.0
+      - gcc-native/12.3
+      - cray-libsci/23.12.5
       - craype-accel-host
-      - craype/2.7.20
-      - libfabric/1.15.2.0
+      - craype/2.7.30
+      - libfabric/1.20.1
       environment:
         prepend_path:
           PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig"
diff --git a/mache/spack/pm-cpu_intel_mpich.csh b/mache/spack/pm-cpu_intel_mpich.csh
index d6eda3d1..52de4034 100644
--- a/mache/spack/pm-cpu_intel_mpich.csh
+++ b/mache/spack/pm-cpu_intel_mpich.csh
@@ -1,38 +1,36 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm gcc-native &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm climate-utils &> /dev/null
-module rm cray-libsci &> /dev/null
-module rm matlab &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-intel/8.5.0
-module load intel/2023.2.0
-module load craype-accel-host
-module load craype/2.7.30
-module load libfabric/1.15.2.0
-module rm cray-mpich &> /dev/null
-module load cray-mpich/8.1.28
+module load PrgEnv-intel/8.5.0 \
+            intel/2023.2.0 \
+            craype-accel-host \
+            craype/2.7.30 \
+            libfabric/1.20.1 \
+            cray-mpich/8.1.28
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.9
-module load cray-netcdf-hdf5parallel/4.9.0.9
-module load cray-parallel-netcdf/1.12.3.9
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -52,6 +50,7 @@ setenv HDF5_USE_FILE_LOCKING FALSE
 ## Not needed
 # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
 setenv FI_CXI_RX_MATCH_MODE software
+setenv FI_MR_CACHE_MONITOR kdreg2
 setenv MPICH_COLL_SYNC MPI_Bcast
 setenv GATOR_INITIAL_MB 4000MB
 setenv BLA_VENDOR Intel10_64_dyn
diff --git a/mache/spack/pm-cpu_intel_mpich.sh b/mache/spack/pm-cpu_intel_mpich.sh
index f997eaca..5d4e3b85 100644
--- a/mache/spack/pm-cpu_intel_mpich.sh
+++ b/mache/spack/pm-cpu_intel_mpich.sh
@@ -1,38 +1,36 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm gcc-native &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm climate-utils &> /dev/null
-module rm cray-libsci &> /dev/null
-module rm matlab &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-intel/8.5.0
-module load intel/2023.2.0
-module load craype-accel-host
-module load craype/2.7.30
-module load libfabric/1.15.2.0
-module rm cray-mpich &> /dev/null
-module load cray-mpich/8.1.28
+module load PrgEnv-intel/8.5.0 \
+            intel/2023.2.0 \
+            craype-accel-host \
+            craype/2.7.30 \
+            libfabric/1.20.1 \
+            cray-mpich/8.1.28
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.9
-module load cray-netcdf-hdf5parallel/4.9.0.9
-module load cray-parallel-netcdf/1.12.3.9
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -52,12 +50,12 @@ export HDF5_USE_FILE_LOCKING=FALSE
 ## Not needed
 # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
 export FI_CXI_RX_MATCH_MODE=software
+export FI_MR_CACHE_MONITOR=kdreg2
 
 if [ -z "${NERSC_HOST:-}" ]; then
   # happens when building spack environment
   export NERSC_HOST="perlmutter"
 fi
-export FI_CXI_RX_MATCH_MODE=software
 export MPICH_COLL_SYNC=MPI_Bcast
 export GATOR_INITIAL_MB=4000MB
 export BLA_VENDOR=Intel10_64_dyn
diff --git a/mache/spack/pm-cpu_intel_mpich.yaml b/mache/spack/pm-cpu_intel_mpich.yaml
index 97846bdd..0a2d34fc 100644
--- a/mache/spack/pm-cpu_intel_mpich.yaml
+++ b/mache/spack/pm-cpu_intel_mpich.yaml
@@ -101,22 +101,14 @@ spack:
         - intel/2023.2.0
         - craype-accel-host
         - craype/2.7.30
-        - libfabric/1.15.2.0
+        - libfabric/1.20.1
       buildable: false
     cray-mpich:
       externals:
       - spec: cray-mpich@8.1.28
-        prefix: /opt/cray/pe/mpich/8.1.28/ofi/intel/19.0
         modules:
         - cray-mpich/8.1.28
-        - libfabric/1.15.2.0
-      buildable: false
-    libfabric:
-      externals:
-      - spec: libfabric@1.15.2.0
-        prefix: /opt/cray/libfabric/1.15.2.0
-        modules:
-        - libfabric/1.15.2.0
+        - libfabric/1.20.1
       buildable: false
 {% if e3sm_hdf5_netcdf %}
     hdf5:
@@ -127,7 +119,7 @@ spack:
     parallel-netcdf:
       externals:
       - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared
-        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/intel/19.0/
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/intel/19.0
       buildable: false
     netcdf-c:
       externals:
@@ -158,7 +150,7 @@ spack:
       - intel/2023.2.0
       - craype-accel-host
       - craype/2.7.30
-      - libfabric/1.15.2.0
+      - libfabric/1.20.1
       environment:
         prepend_path:
           PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig"
diff --git a/mache/spack/pm-cpu_nvidia_mpich.csh b/mache/spack/pm-cpu_nvidia_mpich.csh
index 87ff8220..6a0f0732 100644
--- a/mache/spack/pm-cpu_nvidia_mpich.csh
+++ b/mache/spack/pm-cpu_nvidia_mpich.csh
@@ -1,42 +1,38 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm gcc-native &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm climate-utils &> /dev/null
-module rm cray-libsci &> /dev/null
-module rm matlab &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-nvidia
-module load nvidia/22.7
-module load craype-x86-milan
-module load libfabric/1.15.2.0
-module load craype-accel-host
-module load craype/2.7.20
-module rm cray-mpich &> /dev/null
-module load cray-mpich/8.1.25
-{% if e3sm_lapack %}
-module load cray-libsci/23.02.1.1
-{% endif %}
+module load PrgEnv-nvidia \
+            nvidia/24.5 \
+            cray-libsci/23.12.5 \
+            craype-accel-host \
+            craype/2.7.30 \
+            libfabric/1.20.1 \
+            cray-mpich/8.1.28 \
+            cmake/3.24.3
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -56,6 +52,8 @@ setenv HDF5_USE_FILE_LOCKING FALSE
 ## Not needed
 # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
 setenv MPICH_GPU_SUPPORT_ENABLED 1
+setenv FI_CXI_RX_MATCH_MODE software
+setenv FI_MR_CACHE_MONITOR kdreg2
 setenv MPICH_COLL_SYNC MPI_Bcast
 setenv GATOR_INITIAL_MB 4000MB
 setenv BLA_VENDOR NVHPC
diff --git a/mache/spack/pm-cpu_nvidia_mpich.sh b/mache/spack/pm-cpu_nvidia_mpich.sh
index f28d4d23..90f1d967 100644
--- a/mache/spack/pm-cpu_nvidia_mpich.sh
+++ b/mache/spack/pm-cpu_nvidia_mpich.sh
@@ -1,42 +1,38 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm gcc-native &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm climate-utils &> /dev/null
-module rm cray-libsci &> /dev/null
-module rm matlab &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-nvidia
-module load nvidia/22.7
-module load craype-x86-milan
-module load libfabric/1.15.2.0
-module load craype-accel-host
-module load craype/2.7.20
-module rm cray-mpich &> /dev/null
-module load cray-mpich/8.1.25
-{% if e3sm_lapack %}
-module load cray-libsci/23.02.1.1
-{% endif %}
+module load PrgEnv-nvidia \
+            nvidia/24.5 \
+            cray-libsci/23.12.5 \
+            craype-accel-host \
+            craype/2.7.30 \
+            libfabric/1.20.1 \
+            cray-mpich/8.1.28 \
+            cmake/3.24.3
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -61,6 +57,8 @@ if [ -z "${NERSC_HOST:-}" ]; then
   # happens when building spack environment
   export NERSC_HOST="perlmutter"
 fi
+export FI_CXI_RX_MATCH_MODE=software
+export FI_MR_CACHE_MONITOR=kdreg2
 export MPICH_COLL_SYNC=MPI_Bcast
 export GATOR_INITIAL_MB=4000MB
 export BLA_VENDOR=NVHPC
diff --git a/mache/spack/pm-cpu_nvidia_mpich.yaml b/mache/spack/pm-cpu_nvidia_mpich.yaml
index 52b02d73..92789bee 100644
--- a/mache/spack/pm-cpu_nvidia_mpich.yaml
+++ b/mache/spack/pm-cpu_nvidia_mpich.yaml
@@ -1,9 +1,7 @@
 spack:
   specs:
   - cray-mpich
-{% if e3sm_lapack %}
   - cray-libsci
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
   - hdf5
   - netcdf-c
@@ -15,12 +13,10 @@ spack:
     unify: when_possible
   packages:
     all:
-      compiler: [nvhpc@22.7]
+      compiler: [nvhpc@24.5]
       providers:
-        mpi: [cray-mpich@8.1.25]
-{% if e3sm_lapack %}
-        lapack: [cray-libsci@23.02.1.1]
-{% endif %}
+        mpi: [cray-mpich@8.1.28]
+        lapack: [cray-libsci@23.12.5]
     bzip2:
       externals:
       - spec: bzip2@1.0.6
@@ -100,67 +96,59 @@ spack:
       buildable: false
     cray-mpich:
       externals:
-      - spec: cray-mpich@8.1.25
-        prefix: /opt/cray/pe/mpich/8.1.25/ofi/nvidia/20.7
-        modules:
-        - libfabric/1.15.2.0
-        - cray-mpich/8.1.25
-      buildable: false
-    libfabric:
-      externals:
-      - spec: libfabric@1.15.2.0
-        prefix: /opt/cray/libfabric/1.15.2.0
+      - spec: cray-mpich@8.1.28
         modules:
-        - libfabric/1.15.2.0
+        - libfabric/1.20.1
+        - cray-mpich/8.1.28
       buildable: false
-{% if e3sm_lapack %}
     cray-libsci:
       externals:
-      - spec: cray-libsci@23.02.1.1
-        prefix: /opt/cray/pe/libsci/23.02.1.1/NVIDIA/20.7/x86_64
+      - spec: cray-libsci@23.12.5
+        module:
+        - cray-libsci/23.12.5
       buildable: false
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
     hdf5:
       externals:
-      - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/nvidia/20.7
+      - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/nvidia/23.3
       buildable: false
     parallel-netcdf:
       externals:
-      - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared
-        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/nvidia/20.7
+      - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/nvidia/23.3
       buildable: false
     netcdf-c:
       externals:
-      - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7
+      - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3
       buildable: false
     netcdf-fortran:
       externals:
       - spec: netcdf-fortran@4.5.3
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3
       buildable: false
 {% endif %}
   config:
     install_missing_compilers: false
   compilers:
   - compiler:
-      spec: nvhpc@22.7
+      spec: nvhpc@24.5
       paths:
-        cc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc
-        cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc++
-        f77: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran
-        fc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran
+        cc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc
+        cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc++
+        f77: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran
+        fc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran
       flags: {}
       operating_system: sles15
       target: any
       modules:
       - PrgEnv-nvidia
-      - nvidia/22.7
-      - craype-x86-milan
-      - libfabric
+      - nvidia/24.5
+      - cray-libsci/23.12.5
       - craype-accel-host
+      - craype/2.7.30
+      - libfabric/1.20
       environment:
         prepend_path:
           PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig"
diff --git a/mache/spack/pm-gpu_gnugpu_mpich.csh b/mache/spack/pm-gpu_gnugpu_mpich.csh
index 53e03d55..b13ef5b3 100644
--- a/mache/spack/pm-gpu_gnugpu_mpich.csh
+++ b/mache/spack/pm-gpu_gnugpu_mpich.csh
@@ -1,41 +1,39 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm climate-utils &> /dev/null
-module rm matlab &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-gnu/8.3.3
-module load gcc/11.2.0
-module load craype-x86-milan
-module load libfabric/1.15.2.0
-module load cudatoolkit/11.7
-module load craype-accel-nvidia80
-module load craype/2.7.20
-module rm cray-mpich &> /dev/null
-module load cray-mpich/8.1.25
-{% if e3sm_lapack %}
-module load cray-libsci/23.02.1.1
-{% endif %}
+module load PrgEnv-gnu/8.5.0 \
+            gcc-native/12.3 \
+            cudatoolkit/12.4 \
+            craype-accel-nvidia80 \
+            cray-libsci/23.12.5 \
+            libfabric/1.20.1 \
+            craype/2.7.30 \
+            cray-mpich/8.1.28 \
+            cmake/3.24.3
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -54,4 +52,6 @@ setenv MPICH_MPIIO_DVS_MAXNODES 1
 setenv HDF5_USE_FILE_LOCKING FALSE
 ## Not needed
 # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
+setenv MPICH_COLL_SYNC MPI_Bcast
+setenv FI_MR_CACHE_MONITOR kdreg2
 setenv MPICH_GPU_SUPPORT_ENABLED 1
diff --git a/mache/spack/pm-gpu_gnugpu_mpich.sh b/mache/spack/pm-gpu_gnugpu_mpich.sh
index 3ce26746..009ffd41 100644
--- a/mache/spack/pm-gpu_gnugpu_mpich.sh
+++ b/mache/spack/pm-gpu_gnugpu_mpich.sh
@@ -1,41 +1,39 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm climate-utils &> /dev/null
-module rm matlab &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-gnu/8.3.3
-module load gcc/11.2.0
-module load craype-x86-milan
-module load libfabric/1.15.2.0
-module load cudatoolkit/11.7
-module load craype-accel-nvidia80
-module load craype/2.7.20
-module rm cray-mpich &> /dev/null
-module load cray-mpich/8.1.25
-{% if e3sm_lapack %}
-module load cray-libsci/23.02.1.1
-{% endif %}
+module load PrgEnv-gnu/8.5.0 \
+            gcc-native/12.3 \
+            cudatoolkit/12.4 \
+            craype-accel-nvidia80 \
+            cray-libsci/23.12.5 \
+            libfabric/1.20.1 \
+            craype/2.7.30 \
+            cray-mpich/8.1.28 \
+            cmake/3.24.3
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -54,6 +52,8 @@ export MPICH_VERSION_DISPLAY=1
 export HDF5_USE_FILE_LOCKING=FALSE
 ## Not needed
 # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
+export MPICH_COLL_SYNC=MPI_Bcast
+export FI_MR_CACHE_MONITOR=kdreg2
 export MPICH_GPU_SUPPORT_ENABLED=1
 
 if [ -z "${NERSC_HOST:-}" ]; then
diff --git a/mache/spack/pm-gpu_gnugpu_mpich.yaml b/mache/spack/pm-gpu_gnugpu_mpich.yaml
index 985ceddb..0f18d918 100644
--- a/mache/spack/pm-gpu_gnugpu_mpich.yaml
+++ b/mache/spack/pm-gpu_gnugpu_mpich.yaml
@@ -2,9 +2,7 @@ spack:
   specs:
   - gcc
   - cray-mpich
-{% if e3sm_lapack %}
   - cray-libsci
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
   - hdf5
   - netcdf-c
@@ -16,12 +14,10 @@ spack:
     unify: when_possible
   packages:
     all:
-      compiler: [gcc@11.2.0]
+      compiler: [gcc@12.3]
       providers:
-        mpi: [cray-mpich@8.1.24]
-{% if e3sm_lapack %}
-        lapack: [cray-libsci@23.02.1.1]
-{% endif %}
+        mpi: [cray-mpich@8.1.28]
+        lapack: [cray-libsci@23.12.5]
     bzip2:
       externals:
       - spec: bzip2@1.0.6
@@ -101,78 +97,66 @@ spack:
       buildable: false
     gcc:
       externals:
-      - spec: gcc@11.2.0
+      - spec: gcc@12.3
         modules:
-        - PrgEnv-gnu/8.3.3
-        - gcc/11.2.0
-        - cudatoolkit/11.7
+        - PrgEnv-gnu/8.5.0
+        - gcc-native/12.3
         - craype-accel-nvidia80
-        - craype/2.7.19
-        - libfabric/1.15.2.0
+        - cray-libsci/23.12.5
+        - libfabric/1.20.1
+        - craype/2.7.30
+        - cudatoolkit/12.4
       buildable: false
     cuda:
       externals:
-      - spec: cuda@11.7
-        prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7
+      - spec: cuda@12.4
         modules:
-        - PrgEnv-gnu/8.3.3
-        - gcc/11.2.0
-        - cudatoolkit/11.7
+        - PrgEnv-gnu/8.5.0
+        - gcc-native/12.3
         - craype-accel-nvidia80
-        - craype/2.7.19
-        - libfabric/1.15.2.0
+        - cray-libsci/23.12.5
+        - libfabric/1.20.1
+        - craype/2.7.30
+        - cudatoolkit/12.4
       buildable: false
     cray-mpich:
       externals:
-      - spec: cray-mpich@8.1.25
-        prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1
-        modules:
-        - libfabric/1.15.2.0
-        - cray-mpich/8.1.25
+      - spec: cray-mpich@8.1.28
+        prefix: /opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3
       buildable: false
-    libfabric:
-      externals:
-      - spec: libfabric@1.15.2.0
-        prefix: /opt/cray/libfabric/1.15.2.0
-        modules:
-        - libfabric/1.15.2.0
-      buildable: false
-{% if e3sm_lapack %}
     cray-libsci:
       externals:
-      - spec: cray-libsci@23.02.1.1
-        prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64
+      - spec: cray-libsci@23.12.5
         modules:
-        - cray-libsci/23.02.1.1
+        - cray-libsci/23.12.5
       buildable: false
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
     hdf5:
       externals:
-      - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1
+      - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3
       buildable: false
     parallel-netcdf:
       externals:
-      - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared
-        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/GNU/9.1/
+      - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3
       buildable: false
     netcdf-c:
       externals:
-      - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
+      - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
       buildable: false
     netcdf-fortran:
       externals:
       - spec: netcdf-fortran@4.5.3
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
       buildable: false
 {% endif %}
   config:
     install_missing_compilers: false
   compilers:
   - compiler:
-      spec: gcc@11.2.0
+      spec: gcc@12.3
       paths:
         cc: cc
         cxx: CC
@@ -182,12 +166,13 @@ spack:
       operating_system: sles15
       target: any
       modules:
-      - PrgEnv-gnu/8.3.3
-      - gcc/11.2.0
-      - cudatoolkit/11.7
+      - PrgEnv-gnu/8.5.0
+      - gcc-native/12.3
       - craype-accel-nvidia80
-      - craype/2.7.20
-      - libfabric/1.15.2.0
+      - cray-libsci/23.12.5
+      - libfabric/1.20.1
+      - craype/2.7.30
+      - cudatoolkit/12.4
       environment:
         prepend_path:
           PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig"
diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.csh b/mache/spack/pm-gpu_nvidiagpu_mpich.csh
index 5b474a81..4aca5458 100644
--- a/mache/spack/pm-gpu_nvidiagpu_mpich.csh
+++ b/mache/spack/pm-gpu_nvidiagpu_mpich.csh
@@ -1,42 +1,40 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm climate-utils &> /dev/null
-module rm matlab &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-nvidia
-module load nvidia/22.7
-module load craype-x86-milan
-module load libfabric/1.15.2.0
-module load cudatoolkit/11.7
-module load craype-accel-nvidia80
-module load gcc-mixed/11.2.0
-module load craype/2.7.20
-module rm cray-mpich &> /dev/null
-module load cray-mpich/8.1.25
-{% if e3sm_lapack %}
-module load cray-libsci/23.02.1.1
-{% endif %}
+module load PrgEnv-nvidia \
+            nvidia/24.5 \
+            cudatoolkit/12.4 \
+            craype-accel-nvidia80 \
+            gcc-native-mixed/12.3 \
+            cray-libsci/23.12.5 \
+            libfabric/1.20.1 \
+            craype/2.7.30 \
+            cray-mpich/8.1.28 \
+            cmake/3.24.3
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -53,6 +51,8 @@ setenv MPICH_MPIIO_DVS_MAXNODES 1
 # setenv OMP_PROC_BIND spread
 # setenv OMP_PLACES threads
 setenv HDF5_USE_FILE_LOCKING FALSE
+setenv MPICH_COLL_SYNC MPI_Bcast
+setenv FI_MR_CACHE_MONITOR kdreg2
 ## Not needed
 # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
 setenv MPICH_GPU_SUPPORT_ENABLED 1
diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.sh b/mache/spack/pm-gpu_nvidiagpu_mpich.sh
index ae949ff8..0b88e212 100644
--- a/mache/spack/pm-gpu_nvidiagpu_mpich.sh
+++ b/mache/spack/pm-gpu_nvidiagpu_mpich.sh
@@ -1,42 +1,40 @@
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module rm PrgEnv-gnu &> /dev/null
-module rm PrgEnv-intel &> /dev/null
-module rm PrgEnv-nvidia &> /dev/null
-module rm PrgEnv-cray &> /dev/null
-module rm PrgEnv-aocc &> /dev/null
-module rm intel &> /dev/null
-module rm intel-oneapi &> /dev/null
-module rm cudatoolkit &> /dev/null
-module rm climate-utils &> /dev/null
-module rm matlab &> /dev/null
-module rm craype-accel-nvidia80 &> /dev/null
-module rm craype-accel-host &> /dev/null
-module rm perftools-base &> /dev/null
-module rm perftools &> /dev/null
-module rm darshan &> /dev/null
+module rm cpe \
+          cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          matlab \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          cray-mpich &> /dev/null
 
-module load PrgEnv-nvidia
-module load nvidia/22.7
-module load craype-x86-milan
-module load libfabric/1.15.2.0
-module load cudatoolkit/11.7
-module load craype-accel-nvidia80
-module load gcc-mixed/11.2.0
-module load craype/2.7.20
-module rm cray-mpich &> /dev/null
-module load cray-mpich/8.1.25
-{% if e3sm_lapack %}
-module load cray-libsci/23.02.1.1
-{% endif %}
+module load PrgEnv-nvidia \
+            nvidia/24.5 \
+            cudatoolkit/12.4 \
+            craype-accel-nvidia80 \
+            gcc-native-mixed/12.3 \
+            cray-libsci/23.12.5 \
+            libfabric/1.20.1 \
+            craype/2.7.30 \
+            cray-mpich/8.1.28 \
+            cmake/3.24.3
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel &> /dev/null
-module rm cray-netcdf-hdf5parallel &> /dev/null
-module rm cray-parallel-netcdf &> /dev/null
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 {% if e3sm_hdf5_netcdf %}
@@ -55,6 +53,8 @@ export MPICH_MPIIO_DVS_MAXNODES=1
 export HDF5_USE_FILE_LOCKING=FALSE
 ## Not needed
 # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
+export MPICH_COLL_SYNC=MPI_Bcast
+export FI_MR_CACHE_MONITOR=kdreg2
 export MPICH_GPU_SUPPORT_ENABLED=1
 
 if [ -z "${NERSC_HOST:-}" ]; then
diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml
index 5c1338df..863652f5 100644
--- a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml
+++ b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml
@@ -1,9 +1,7 @@
 spack:
   specs:
   - cray-mpich
-{% if e3sm_lapack %}
   - cray-libsci
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
   - hdf5
   - netcdf-c
@@ -15,12 +13,10 @@ spack:
     unify: when_possible
   packages:
     all:
-      compiler: [nvhpc@22.7]
+      compiler: [nvhpc@24.5]
       providers:
-        mpi: [cray-mpich@8.1.25]
-{% if e3sm_lapack %}
-        lapack: [cray-libsci@23.02.1.1]
-{% endif %}
+        mpi: [cray-mpich@8.1.28]
+        lapack: [cray-libsci@23.12.5]
     bzip2:
       externals:
       - spec: bzip2@1.0.6
@@ -100,84 +96,72 @@ spack:
       buildable: false
     cuda:
       externals:
-      - spec: cuda@11.7
-        prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7
+      - spec: cuda@12.4
         modules:
         - PrgEnv-nvidia
-        - nvidia/22.7
-        - cudatoolkit/11.7
+        - nvidia/24.5
         - craype-accel-nvidia80
-        - gcc-mixed/11.2.0
-        - craype-x86-milan
-        - libfabric
+        - cray-libsci/23.12.5
+        - libfabric/1.20.1
+        - craype/2.7.30
+        - cudatoolkit/12.4
       buildable: false
     cray-mpich:
       externals:
-      - spec: cray-mpich@8.1.25
-        prefix: /opt/cray/pe/mpich/8.1.25/ofi/nvidia/20.7
-        modules:
-        - libfabric/1.15.2.0
-        - cray-mpich/8.1.25
-      buildable: false
-    libfabric:
-      externals:
-      - spec: libfabric@1.15.2.0
-        prefix: /opt/cray/libfabric/1.15.2.0
+      - spec: cray-mpich@8.1.28
         modules:
-        - libfabric/1.15.2.0
+        - libfabric/1.20.1
+        - cray-mpich/8.1.28
       buildable: false
-{% if e3sm_lapack %}
     cray-libsci:
       externals:
-      - spec: cray-libsci@23.02.1.1
-        prefix: /opt/cray/pe/libsci/23.02.1.1/NVIDIA/20.7/x86_64
+      - spec: cray-libsci@23.12.5
         modules:
-        - cray-libsci/23.02.1.1
+        - cray-libsci/23.12.5
       buildable: false
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
     hdf5:
       externals:
-      - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/nvidia/20.7
+      - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/nvidia/23.3
       buildable: false
     parallel-netcdf:
       externals:
-      - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared
-        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/nvidia/20.7
+      - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/nvidia/23.3
       buildable: false
     netcdf-c:
       externals:
-      - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7
+      - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3
       buildable: false
     netcdf-fortran:
       externals:
       - spec: netcdf-fortran@4.5.3
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3
       buildable: false
 {% endif %}
   config:
     install_missing_compilers: false
   compilers:
   - compiler:
-      spec: nvhpc@22.7
+      spec: nvhpc@24.5
       paths:
-        cc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc
-        cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc++
-        f77: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran
-        fc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran
+        cc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc
+        cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc++
+        f77: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran
+        fc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran
       flags: {}
       operating_system: sles15
       target: any
       modules:
       - PrgEnv-nvidia
-      - nvidia/22.7
-      - cudatoolkit/11.7
+      - nvidia/24.5
       - craype-accel-nvidia80
-      - gcc-mixed/11.2.0
-      - craype-x86-milan
-      - libfabric
+      - cray-libsci/23.12.5
+      - libfabric/1.20.1
+      - craype/2.7.30
+      - cudatoolkit/12.4
       environment:
         prepend_path:
           PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig"