Skip to content

Commit

Permalink
Merge branch E3SM-Project/ndk/machinefiles/alvarez-update (PR #5966)
Browse files Browse the repository at this point in the history
For alvarez (internal test machine for pm-cpu), update machine file settings
  • Loading branch information
ndkeen authored Oct 4, 2023
2 parents 894b5b2 + 1f42fb8 commit 573137b
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 13 deletions.
14 changes: 14 additions & 0 deletions cime_config/machines/Depends.pm-cpu.alvarez.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# For this file, fixes non-BFB behavior of stealth feature on pm-cpu with -O2
set(NOOPT
eam/src/physics/cam/zm_conv.F90)

if (NOT DEBUG)
foreach(ITEM IN LISTS NOOPT)
e3sm_remove_flags("${ITEM}" "-O2")
e3sm_remove_flags("${ITEM}" "-O")
endforeach()
endif()




49 changes: 49 additions & 0 deletions cime_config/machines/cmake_macros/intel_alvarez.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
string(APPEND CONFIG_ARGS " --host=cray")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
endif()
string(APPEND SLIBS " -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}/lib -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff")
#if (NOT MPILIB STREQUAL mpi-serial)
# string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
#endif()
string(APPEND SLIBS " -qmkl")
set(CXX_LINKER "FORTRAN")
set(NETCDF_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}")
set(NETCDF_C_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}")
set(NETCDF_FORTRAN_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}")
set(HDF5_PATH "$ENV{CRAY_HDF5_PARALLEL_PREFIX}")
set(PNETCDF_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}")

set(MPICC "cc")
set(MPICXX "CC")
set(MPIFC "ftn")
set(SCC "icx")
set(SCXX "icpx")
set(SFC "ifx")

# Bit of a hack here. For whatever reason, the intel version on pm-cpu (both intel and intel-oneapi, and both icpc/icpx)
# does not seem to have the -fp-model=source flag (docs still show it). And I was unable to find a reliable way of testing
# on the compiler ID or version, so for now, simply manually adjust the CXXFLAG setting for pm-cpu/intel
# Try to manually remove -fp-model=source (and replace with -fp-model=precise) from CXXFLAGS
#message(STATUS "ndk CXXFLAGS=${CXXFLAGS}")
set(CXXFLAGS " ") # hardcode it here to blank, then try to do same things as in intel.cmake
if (compile_threaded)
string(APPEND CXXFLAGS " -qopenmp")
endif()
if (DEBUG)
string(APPEND CXXFLAGS " -O0 -g")
endif()
if (NOT DEBUG)
string(APPEND CXXFLAGS " -O2")
endif()
string(APPEND CXXFLAGS " -fp-model=precise") # and manually add precise
#message(STATUS "ndk CXXFLAGS=${CXXFLAGS}")

string(APPEND FFLAGS " -fp-model=consistent -fimf-use-svml")
if (NOT DEBUG)
# string(APPEND FFLAGS " -qno-opt-dynamic-align")
string(APPEND FFLAGS " -g -traceback")
string(APPEND CXXFLAGS " -g -traceback")
endif()
string(APPEND FFLAGS " -DHAVE_ERF_INTRINSICS")
string(APPEND CXXFLAGS " -fp-model=consistent")
37 changes: 24 additions & 13 deletions cime_config/machines/config_machines.xml
Original file line number Diff line number Diff line change
Expand Up @@ -413,12 +413,12 @@
<DESC>test machine at NERSC, very similar to pm-cpu. each node has 2 AMD EPYC 7713 64-Core (Milan) 512GB</DESC>
<NODENAME_REGEX>$ENV{NERSC_HOST}:alvarez</NODENAME_REGEX>
<OS>Linux</OS>
<COMPILERS>gnu,nvidia,amdclang</COMPILERS>
<COMPILERS>intel,gnu,nvidia,amdclang</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<PROJECT>e3sm</PROJECT>
<SAVE_TIMING_DIR>/global/cfs/cdirs/e3sm</SAVE_TIMING_DIR>
<SAVE_TIMING_DIR_PROJECTS>e3sm,m3411,m3412</SAVE_TIMING_DIR_PROJECTS>
<CIME_OUTPUT_ROOT>$ENV{PSCRATCH}/e3sm_scratch/alvarez</CIME_OUTPUT_ROOT>
<CIME_OUTPUT_ROOT>$ENV{SCRATCH}/e3sm_scratch/alvarez</CIME_OUTPUT_ROOT>
<CIME_HTML_ROOT>/global/cfs/cdirs/e3sm/www/$ENV{USER}</CIME_HTML_ROOT>
<CIME_URL_ROOT>http://portal.nersc.gov/project/e3sm/$ENV{USER}</CIME_URL_ROOT>
<DIN_LOC_ROOT>/global/cfs/cdirs/e3sm/inputdata</DIN_LOC_ROOT>
Expand All @@ -445,12 +445,12 @@
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="perl">/usr/share/lmod/8.3.1/init/perl</init_path>
<init_path lang="python">/usr/share/lmod/8.3.1/init/python</init_path>
<init_path lang="sh">/usr/share/lmod/8.3.1/init/sh</init_path>
<init_path lang="csh">/usr/share/lmod/8.3.1/init/csh</init_path>
<cmd_path lang="perl">/usr/share/lmod/lmod/libexec/lmod perl</cmd_path>
<cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
<init_path lang="perl">/opt/cray/pe/lmod/8.7.19/init/perl</init_path>
<init_path lang="python">/opt/cray/pe/lmod/8.7.19/init/python</init_path>
<init_path lang="sh">/opt/cray/pe/lmod/8.7.19/init/sh</init_path>
<init_path lang="csh">/opt/cray/pe/lmod/8.7.19/init/csh</init_path>
<cmd_path lang="perl">/opt/cray/pe/lmod/lmod/libexec/lmod perl</cmd_path>
<cmd_path lang="python">/opt/cray/pe/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>

Expand All @@ -459,9 +459,12 @@
<command name="unload">cray-netcdf-hdf5parallel</command>
<command name="unload">cray-parallel-netcdf</command>
<command name="unload">PrgEnv-gnu</command>
<command name="unload">PrgEnv-intel</command>
<command name="unload">PrgEnv-nvidia</command>
<command name="unload">PrgEnv-cray</command>
<command name="unload">PrgEnv-aocc</command>
<command name="unload">intel</command>
<command name="unload">intel-oneapi</command>
<command name="unload">cudatoolkit</command>
<command name="unload">craype-accel-nvidia80</command>
<command name="unload">craype-accel-host</command>
Expand All @@ -471,25 +474,33 @@
</modules>

<modules compiler="gnu">
<command name="load">PrgEnv-gnu/8.3.3</command>
<command name="load">gcc/12.1.0</command>
<command name="load">PrgEnv-gnu</command>
<command name="load">gcc</command>
<command name="load">cray-libsci</command>
</modules>

<modules compiler="intel">
<command name="load">PrgEnv-intel</command>
<command name="load">intel</command>
</modules>

<modules compiler="nvidia">
<command name="load">PrgEnv-nvidia</command>
<command name="load">nvidia/22.7</command>
<command name="load">cray-libsci</command>
</modules>

<modules compiler="amdclang">
<command name="load">PrgEnv-aocc</command>
<command name="load">aocc/3.2.0</command>
<command name="load">cray-libsci</command>
</modules>

<modules>
<command name="load">craype-accel-host</command>
<command name="load">cray-libsci/23.02.1.1</command>
<command name="load">craype/2.7.19</command>
<command name="load">cray-mpich/8.1.24</command>
<command name="load">cray-libsci</command>
<command name="load">craype/2.7.21</command>
<command name="load">cray-mpich/8.1.26</command>
<command name="load">cray-hdf5-parallel/1.12.2.3</command>
<command name="load">cray-netcdf-hdf5parallel/4.9.0.3</command>
<command name="load">cray-parallel-netcdf/1.12.3.3</command>
Expand Down

0 comments on commit 573137b

Please sign in to comment.