Skip to content

Commit

Permalink
Update paths and modules, add gnu config
Browse files Browse the repository at this point in the history
  • Loading branch information
@amametjanov
amametjanov committed Jan 25, 2024
1 parent 66d49a5 commit c791365
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 31 deletions.
10 changes: 10 additions & 0 deletions cime_config/machines/cmake_macros/gnu_polaris.cmake
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
string(APPEND CONFIG_ARGS " --host=cray")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
endif()
set(MPICC "cc")
set(MPICXX "CC")
set(MPIFC "ftn")
set(SCC "gcc")
set(SCXX "g++")
set(SFC "gfortran")
10 changes: 6 additions & 4 deletions cime_config/machines/config_batch.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -551,11 +551,13 @@
</batch_system>

<batch_system MACH="polaris" type="pbspro">
<batch_submit>qsub</batch_submit>
<directives>
<directive> -l filesystems=home:grand:eagle </directive>
</directives>
<queues>
<queue walltimemax="01:00:00" nodemin="1" nodemax="2" strict="true" default="true">debug</queue>
<queue walltimemax="01:00:00" nodemin="3" nodemax="9" strict="true">debug-scaling</queue>
<queue walltimemax="02:00:00" nodemin="10" nodemax="496" strict="true">prod</queue>
<queue walltimemax="00:30:00" nodemin="1" nodemax="2" strict="true" default="true">debug</queue>
<queue walltimemax="00:30:00" nodemin="3" nodemax="9" strict="true">debug-scaling</queue>
<queue walltimemax="01:00:00" nodemin="10" nodemax="496" strict="true">prod</queue>
</queues>
</batch_system>

Expand Down
47 changes: 20 additions & 27 deletions cime_config/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2961,25 +2961,25 @@
<OS>Linux</OS>
<COMPILERS>gnu,gnugpu,nvidia,nvidiagpu</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<PROJECT>SCREAM_Calib</PROJECT>
<CHARGE_ACCOUNT>SCREAM_Calib</CHARGE_ACCOUNT>
<SAVE_TIMING_DIR>/lus/grand/projects/SCREAM_Calib/E3SM/performance_archive</SAVE_TIMING_DIR>
<SAVE_TIMING_DIR_PROJECTS>SCREAM_Calib</SAVE_TIMING_DIR_PROJECTS>
<CIME_OUTPUT_ROOT>/lus/grand/projects/SCREAM_Calib/E3SM/$USER/scratch</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/lus/grand/projects/SCREAM_Calib/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/lus/grand/projects/SCREAM_Calib/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<PROJECT>E3SM_RRM</PROJECT>
<CHARGE_ACCOUNT>E3SM_RRM</CHARGE_ACCOUNT>
<SAVE_TIMING_DIR>/eagle/E3SM_RRM/performance_archive</SAVE_TIMING_DIR>
<SAVE_TIMING_DIR_PROJECTS>SCREAM_Calib,E3SM_RRM</SAVE_TIMING_DIR_PROJECTS>
<CIME_OUTPUT_ROOT>/eagle/E3SM_RRM/$USER/scratch</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/grand/SCREAM_Calib/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/grand/SCREAM_Calib/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/lus/grand/projects/SCREAM_Calib/E3SM/baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/lus/grand/projects/projects/SCREAM_Calib/E3SM/tools/cprnc/cprnc</CCSM_CPRNC>
<GMAKE_J>32</GMAKE_J>
<BASELINE_ROOT>/eagle/E3SM_RRM/baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/eagle/E3SM_RRM/tools/cprnc/cprnc</CCSM_CPRNC>
<GMAKE_J>4</GMAKE_J>
<TESTS>e3sm_developer</TESTS>
<NTEST_PARALLEL_JOBS>4</NTEST_PARALLEL_JOBS>
<BATCH_SYSTEM>pbspro</BATCH_SYSTEM>
<SUPPORTED_BY>e3sm</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>4</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="gnu">64</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="nvidia">64</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="gnu">32</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="nvidia">32</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>mpiexec</executable>
Expand All @@ -3000,10 +3000,11 @@
<modules>
<command name="purge"/>
<command name="load">cmake/3.23.2</command>
<command name="unload">PrgEnv-nvhpc nvhpc craype-accel-nvidia80</command>
</modules>
<modules compiler="gnu">
<command name="load">PrgEnv-gnu/8.3.3</command>
<command name="load">craype-accel-host</command>
<command name="swap">gcc/12.2.0 gcc/11.2.0</command>
</modules>
<modules compiler="gnugpu">
<command name="load">PrgEnv-gnu/8.3.3</command>
Expand All @@ -3019,32 +3020,24 @@
<!--command name="load">cudatoolkit-standalone/11.4.4</command-->
</modules>
<modules>
<command name="load">craype/2.7.15</command>
<command name="load">cray-pals/1.1.7</command>
<command name="load">cray-libsci/21.08.1.2</command>
<command name="load">cray-hdf5-parallel/1.12.1.3</command>
<command name="load">cray-netcdf-hdf5parallel/4.8.1.3</command>
<command name="load">cray-parallel-netcdf/1.12.2.3</command>
<command name="load">craype-network-ofi</command>
<command name="load">libfabric/1.15.2.0</command>
<command name="load">cray-libsci/23.02.1.1</command>
<command name="load">cray-hdf5-parallel/1.12.2.3</command>
<command name="load">cray-netcdf-hdf5parallel/4.9.0.3</command>
<command name="load">cray-parallel-netcdf/1.12.3.3</command>
</modules>
</module_system>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables>
<env name="NETCDF_PATH">$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}</env>
<env name="PNETCDF_PATH">$ENV{CRAY_PARALLEL_NETCDF_PREFIX}</env>
<env name="HDF5_USE_FILE_LOCKING">FALSE</env>
<env name="MPICH_COLL_SYNC">MPI_Bcast</env>
<env name="OPENBLAS_PATH">/grand/SCREAM_Calib/lib/lapack/polaris_build</env>
<env name="LAPACK_PATH">/grand/SCREAM_Calib/lib/lapack/polaris_build</env>
<env name="LD_LIBRARY_PATH">/grand/SCREAM_Calib/lib/lapack/polaris_build/lib:$ENV{LD_LIBRARY_PATH}</env>
</environment_variables>
<environment_variables compiler=".*gpu">
<env name="MPICH_GPU_SUPPORT_ENABLED">1</env>
<env name="CUDA_VISIBLE_DEVICE">1</env>
</environment_variables>
<environment_variables compiler="gnu.*">
<env name="LD_PRELOAD">/opt/cray/pe/gcc/11.2.0/snos/lib64/libstdc++.so.6</env>
</environment_variables>
<environment_variables SMP_PRESENT="TRUE">
<env name="OMP_STACKSIZE">128M</env>
<env name="OMP_PROC_BIND">spread</env>
Expand Down

0 comments on commit c791365

Please sign in to comment.