Fix e3sm_integration tests with gnu

E3SM-Project · Feb 23, 2024 · 5971d1e · 5971d1e
1 parent b70e090
commit 5971d1e
Show file tree

Hide file tree

Showing 8 changed files with 88 additions and 26 deletions.
diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml
@@ -1379,6 +1379,34 @@
         </nthrds>
       </pes>
     </mach>
+    <mach name="improv">
+      <pes compset="any" pesize="any">
+        <comment>improv: any compset on ne30np4 grid</comment>
+        <ntasks>
+          <ntasks_atm>-4</ntasks_atm>
+          <ntasks_lnd>-4</ntasks_lnd>
+          <ntasks_rof>-4</ntasks_rof>
+          <ntasks_ice>-4</ntasks_ice>
+          <ntasks_ocn>-4</ntasks_ocn>
+          <ntasks_cpl>-4</ntasks_cpl>
+          <ntasks_glc>-4</ntasks_glc>
+          <ntasks_wav>-4</ntasks_wav>
+        </ntasks>
+      </pes>
+      <pes compset=".*EAM.+ELM.+MPASSI.+SGLC.+SWAV.+BGC.*" pesize="any">
+        <comment>improv: BGC compset on ne30np4 grid</comment>
+        <ntasks>
+          <ntasks_atm>-8</ntasks_atm>
+          <ntasks_lnd>-8</ntasks_lnd>
+          <ntasks_rof>-8</ntasks_rof>
+          <ntasks_ice>-8</ntasks_ice>
+          <ntasks_ocn>-8</ntasks_ocn>
+          <ntasks_cpl>-8</ntasks_cpl>
+          <ntasks_glc>-8</ntasks_glc>
+          <ntasks_wav>-8</ntasks_wav>
+        </ntasks>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%ne30np4.pg2_l%.+_oi%oEC60to30v3">
     <mach name="compy">

diff --git a/cime_config/machines/Depends.oneapi-ifx.cmake b/cime_config/machines/Depends.oneapi-ifx.cmake
@@ -1,3 +1,5 @@
 
 # compile mpas_seaice_core_interface.f90 with ifort, not ifx
-e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort")
+if (NOT MPILIB STREQUAL "openmpi")
+  e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort")
+endif()
diff --git a/cime_config/machines/cmake_macros/gnu_improv.cmake b/cime_config/machines/cmake_macros/gnu_improv.cmake
@@ -0,0 +1,8 @@
+if (COMP_NAME STREQUAL gptl)
+  string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
+endif()
+string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
+set(PIO_FILESYSTEM_HINTS "gpfs")
+string(APPEND SLIBS " /lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0/liblapack.a /lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0/libblas.a")
+
diff --git a/cime_config/machines/cmake_macros/oneapi-ifx_improv.cmake b/cime_config/machines/cmake_macros/oneapi-ifx_improv.cmake
@@ -1,14 +1,10 @@
 if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
 endif()
-if (NOT DEBUG)
-  string(APPEND CFLAGS   " -O3")
-  string(APPEND CXXFLAGS " -O3")
-  string(APPEND FFLAGS   " -O3 -qno-opt-dynamic-align")
-endif()
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -qno-opt-dynamic-align")
 set(PIO_FILESYSTEM_HINTS "gpfs")
 if (MPILIB STREQUAL impi)
-  set(MPICC "mpiicx")
+  set(MPICC  "mpiicx")
   set(MPICXX "mpiicpx")
-  set(MPIFC "mpiifx")
+  set(MPIFC  "mpiifx")
 endif()
diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml
@@ -428,8 +428,8 @@
       <directive> -l select={{ num_nodes }}:mpiprocs={{ tasks_per_node }} </directive>
     </directives>
     <queues>
-      <queue walltimemax="01:00:00" jobmin="1" jobmax="8" strict="true" default="true">debug</queue>
-      <queue walltimemax="01:00:00" jobmin="9" jobmax="737" strict="true">compute</queue>
+      <queue walltimemax="00:30:00" jobmin="1" jobmax="8" strict="true" default="true">debug</queue>
+      <queue walltimemax="00:30:00" jobmin="9" jobmax="737" strict="true">compute</queue>
     </queues>
   </batch_system>
 

diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
@@ -2744,7 +2744,7 @@
     <CCSM_CPRNC>/lcrc/group/e3sm/tools/cprnc/cprnc.improv</CCSM_CPRNC>
     <GMAKE_J>8</GMAKE_J>
     <TESTS>e3sm_integration</TESTS>
-    <NTEST_PARALLEL_JOBS>4</NTEST_PARALLEL_JOBS>
+    <NTEST_PARALLEL_JOBS>8</NTEST_PARALLEL_JOBS>
     <BATCH_SYSTEM>pbspro</BATCH_SYSTEM>
     <SUPPORTED_BY>E3SM</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
@@ -2795,23 +2795,14 @@
         <command name="load">parallel-netcdf/1.12.3-intel-oneapi-mpi-2021.10.0-oneapi-2023.2.1</command>
       </modules>
       <modules compiler="gnu">
-        <command name="load">gcc/13.2.0</command>
-        <command name="load">intel-mkl/2020.4.304-gcc-13.2.0</command>
-      </modules>
-      <modules compiler="gnu" mpilib="openmpi">
-        <command name="load">openmpi/4.1.6-gcc-13.2.0</command>
-        <command name="load">hdf5/1.14.3-openmpi-4.1.6-gcc-13.2.0</command>
-        <command name="load">netcdf-c/4.9.2-openmpi-4.1.6-gcc-13.2.0</command>
-        <command name="load">netcdf-fortran/4.6.1-openmpi-4.1.6-gcc-13.2.0</command>
-        <command name="load">parallel-netcdf/1.12.3-openmpi-4.1.6-gcc-13.2.0</command>
+        <command name="load">gcc/12.3.0</command>
       </modules>
     </module_system>
     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
     <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
     <TEST_TPUT_TOLERANCE>0.05</TEST_TPUT_TOLERANCE>
-    <TEST_MEMLEAK_TOLERANCE>0.05</TEST_MEMLEAK_TOLERANCE>
     <MAX_GB_OLD_TEST_DATA>0</MAX_GB_OLD_TEST_DATA>
-    <environment_variables>
+    <environment_variables compiler="!gnu">
       <env name="NETCDF_C_PATH">$SHELL{dirname $(dirname $(which nc-config))}</env>
       <env name="NETCDF_FORTRAN_PATH">$SHELL{dirname $(dirname $(which nf-config))}</env>
       <env name="PNETCDF_PATH">$SHELL{dirname $(dirname $(which pnetcdf_version))}</env>
@@ -2825,6 +2816,13 @@
     <environment_variables mpilib="impi" DEBUG="TRUE">
       <env name="I_MPI_DEBUG">10</env>
     </environment_variables>
+    <environment_variables compiler="gnu" mpilib="openmpi">
+      <env name="NETCDF_C_PATH">/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6</env>
+      <env name="NETCDF_FORTRAN_PATH">/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6</env>
+      <env name="PNETCDF_PATH">/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6</env>
+      <env name="PATH">/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH}</env>
+      <env name="LD_LIBRARY_PATH">$SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi}</env>
+    </environment_variables>
     <environment_variables BUILD_THREADED="TRUE">
       <env name="OMP_STACKSIZE">128M</env>
     </environment_variables>

diff --git a/components/eam/cime_config/config_pes.xml b/components/eam/cime_config/config_pes.xml
@@ -756,6 +756,21 @@
         </nthrds>
       </pes>
     </mach>
+    <mach name="improv">
+      <pes compset=".*EAM.+ELM.+MPASSI.+DOCN.+SGLC_SWAV_SIAC_SESP_BGC.*" pesize="any">
+        <comment>improv pelayout for tri-grid BGC tests with EAM+DOCN </comment>
+        <ntasks>
+          <ntasks_atm>-4</ntasks_atm>
+          <ntasks_cpl>-4</ntasks_cpl>
+          <ntasks_lnd>-4</ntasks_lnd>
+          <ntasks_rof>-4</ntasks_rof>
+          <ntasks_ocn>-4</ntasks_ocn>
+          <ntasks_ice>-4</ntasks_ice>
+          <ntasks_glc>-4</ntasks_glc>
+          <ntasks_wav>-4</ntasks_wav>
+        </ntasks>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%ne30np4.pg2_l%.+_oi%oEC60to30v3">
     <mach name="anvil">
@@ -1062,6 +1077,21 @@
         </ntasks>
       </pes>
     </mach>
+    <mach name="improv">
+      <pes compset=".*EAM.+ELM.+MPASSI.+DOCN.+MOSART.+" pesize="M">
+        <comment>--res conusx4v1_r05_oECv3 --compset F2010 </comment>
+        <ntasks>
+          <ntasks_atm>-4</ntasks_atm>
+          <ntasks_cpl>-4</ntasks_cpl>
+          <ntasks_lnd>-4</ntasks_lnd>
+          <ntasks_rof>-4</ntasks_rof>
+          <ntasks_ocn>-4</ntasks_ocn>
+          <ntasks_ice>-4</ntasks_ice>
+          <ntasks_glc>-4</ntasks_glc>
+          <ntasks_wav>-4</ntasks_wav>
+        </ntasks>
+      </pes>
+    </mach>
     <mach name="summit|ascent">
       <pes compset=".*EAM.+ELM.+MPASSI.+DOCN.+MOSART.+" pesize="M">
         <comment>--res conusx4v1_r05_oECv3 --compset F2010 </comment>

diff --git a/components/elm/cime_config/config_pes.xml b/components/elm/cime_config/config_pes.xml
@@ -218,9 +218,9 @@
         </ntasks>
       </pes>
     </mach>
-    <mach name="ascent|summit">
+    <mach name="ascent|summit|improv">
       <pes compset="any" pesize="any">
-        <comment>elm: ascent|summit PEs for grid l%1.9x2.5|l%0.9x1.25|l%360x720cru</comment>
+        <comment>elm: ascent|summit|improv PEs for grid l%1.9x2.5|l%0.9x1.25|l%360x720cru</comment>
         <ntasks>
           <ntasks_atm>-2</ntasks_atm>
           <ntasks_lnd>-2</ntasks_lnd>
@@ -509,9 +509,9 @@
         </ntasks>
       </pes>
     </mach>
-    <mach name="ascent|summit">
+    <mach name="ascent|summit|improv">
       <pes compset="any" pesize="any">
-        <comment>elm: ascent|summit PEs for grid l%r05_*r%r05</comment>
+        <comment>elm: ascent|summit|improv PEs for grid l%r05_*r%r05</comment>
         <ntasks>
           <ntasks_atm>-2</ntasks_atm>
           <ntasks_lnd>-2</ntasks_lnd>