Update gnugpu, nvidia and nvidiagpu cmake and modules

E3SM-Project · Jan 31, 2024 · 359013d · 359013d
1 parent 74a78e7
commit 359013d
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 83 deletions.
diff --git a/cime_config/machines/cmake_macros/gnugpu_polaris.cmake b/cime_config/machines/cmake_macros/gnugpu_polaris.cmake
@@ -1,35 +1,14 @@
-if (NOT DEBUG)
-  string(APPEND CFLAGS " -O2")
-  string(APPEND CXXFLAGS " -O2")
-  string(APPEND FFLAGS " -O2")
-  string(APPEND CUDA_FLAGS " -gpu -O3 -arch sm_80 --use_fast_math")
-endif()
-if (DEBUG)
-  string(APPEND CFLAGS " -O2 -g")
-  string(APPEND CXXFLAGS " -O2 -g")
-  string(APPEND FFLAGS " -O2 -g")	
-  string(APPEND CUDA_FLAGS " -O0 -g -gpu -arch sm_80 --use_fast_math")
-endif()
-set(USE_CUDA "TRUE")
 string(APPEND CONFIG_ARGS " --host=cray")
 if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
 endif()
-string(APPEND SLIBS " -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff")
-string(APPEND SLIBS " -L$ENV{LAPACK_PATH}/lib -llapack -L$ENV{OPENBLAS_PATH}/lib -lblas")
-string(APPEND SLIBS " $ENV{PE_MPICH_GTL_DIR_nvidia80} $ENV{PE_MPICH_GTL_LIBS_nvidia80}")
-if (NOT MPILIB STREQUAL mpi-serial)
-#  string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
-endif()
-set(CXX_LINKER "FORTRAN")
-set(NETCDF_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}")
-set(NETCDF_C_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}")
-set(NETCDF_FORTRAN_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}")
-set(HDF5_PATH "$ENV{CRAY_HDF5_PARALLEL_PREFIX}")
-set(PNETCDF_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}")
-if (compile_threaded)
-  string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_OPENMP=Off") 
-endif()
+set(USE_CUDA "TRUE")
+string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
+string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_AMPERE80=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=Off")
+set(E3SM_LINK_WITH_FORTRAN "TRUE")
 set(MPICC "cc")
 set(MPICXX "CC")
 set(MPIFC "ftn")

diff --git a/cime_config/machines/cmake_macros/nvhpcgpu_polaris.cmake b/cime_config/machines/cmake_macros/nvhpcgpu_polaris.cmake
diff --git a/cime_config/machines/cmake_macros/nvidia_polaris.cmake b/cime_config/machines/cmake_macros/nvidia_polaris.cmake
@@ -0,0 +1,7 @@
+string(APPEND CONFIG_ARGS " --host=cray")
+if (COMP_NAME STREQUAL gptl)
+  string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
+endif()
+set(SCC "cc")
+set(SCXX "CC")
+set(SFC "ftn")
diff --git a/cime_config/machines/cmake_macros/nvidiagpu_polaris.cmake b/cime_config/machines/cmake_macros/nvidiagpu_polaris.cmake
@@ -1,26 +1,13 @@
 string(APPEND CONFIG_ARGS " --host=cray")
-set(USE_CUDA "TRUE")
-string(APPEND CPPDEFS " -DGPU")
 if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
 endif()
+set(USE_CUDA "TRUE")
+string(APPEND CPPDEFS " -DGPU -DMPAS_OPENACC")
 string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
-string(APPEND CUDA_FLAGS " -ccbin cc -O2 -arch sm_80 --use_fast_math")
-string(APPEND SLIBS " -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff")
-string(APPEND SLIBS " -L$ENV{LAPACK_PATH}/lib -llapack -L$ENV{OPENBLAS_PATH}/lib -lblas")
-if (NOT MPILIB STREQUAL mpi-serial)
-#  string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
-endif()
-set(CXX_LINKER "FORTRAN")
-set(SUPPORTS_CXX "TRUE")
-set(NETCDF_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}")
-set(NETCDF_C_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}")
-set(NETCDF_FORTRAN_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}")
-set(HDF5_PATH "$ENV{CRAY_HDF5_PARALLEL_PREFIX}")
-set(PNETCDF_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}")
+string(APPEND CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc=gpu -gpu=cc80,cuda11.0 -Minfo=accel")
 set(SCC "cc")
 set(SCXX "CC")
 set(SFC "ftn")
-set(MPICC "cc")
-set(MPICXX "CC")
-set(MPIFC "ftn")
+string(APPEND CMAKE_Fortran_FLAGS " -acc=gpu -gpu=cc80,cuda11.0 -Minfo=accel")
diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml
@@ -561,6 +561,7 @@
     </batch_system>
 
     <batch_system MACH="polaris" type="pbspro">
+      <batch_submit>/eagle/E3SM_RRM/tools/qsub/throttle</batch_submit>
       <directives>
         <directive> -l filesystems=home:grand:eagle </directive>
       </directives>

diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
@@ -3180,24 +3180,21 @@
       <modules>
         <command name="purge"/>
         <command name="load">cmake/3.23.2</command>
-        <command name="unload">PrgEnv-nvhpc nvhpc craype-accel-nvidia80</command>
+        <command name="load">craype-x86-rome</command>
       </modules>
-      <modules compiler="gnu">
+      <modules compiler="gnu.*">
         <command name="load">PrgEnv-gnu/8.3.3</command>
-        <command name="swap">gcc/12.2.0 gcc/11.2.0</command>
       </modules>
       <modules compiler="gnugpu">
-        <command name="load">PrgEnv-gnu/8.3.3</command>
         <command name="swap">gcc/12.2.0 gcc/11.2.0</command>
         <command name="load">cudatoolkit-standalone/11.4.4</command>
       </modules>
-      <modules compiler="nvidia">
-        <command name="load">PrgEnv-nvidia/8.3.3</command>
-        <command name="load">craype-accel-host</command>
+      <modules compiler="nvidia.*">
+        <command name="load">PrgEnv-nvhpc/8.3.3</command>
       </modules>
       <modules compiler="nvidiagpu">
-        <command name="load">PrgEnv-nvidia/8.3.3</command>
-        <!--command name="load">cudatoolkit-standalone/11.4.4</command-->
+        <command name="load">cudatoolkit-standalone/11.4.4</command>
+        <command name="load">craype-accel-nvidia80</command>
       </modules>
       <modules>
         <command name="load">craype-network-ofi</command>
@@ -3216,9 +3213,13 @@
     </environment_variables>
     <environment_variables compiler=".*gpu">
       <env name="MPICH_GPU_SUPPORT_ENABLED">1</env>
+      <env name="CRAY_ACCEL_TARGET">nvidia80</env>
       <env name="CUDA_VISIBLE_DEVICE">1</env>
     </environment_variables>
-    <environment_variables SMP_PRESENT="TRUE">
+    <environment_variables compiler="gnu.*">
+      <env name="LD_PRELOAD">/opt/cray/pe/gcc/11.2.0/snos/lib64/libstdc++.so</env>
+    </environment_variables>
+    <environment_variables BUILD_THREADED="TRUE">
       <env name="OMP_STACKSIZE">128M</env>
       <env name="OMP_PROC_BIND">spread</env>
       <env name="OMP_PLACES">threads</env>