From 2e48c25018a0bb171e7276aa402056f7315b902e Mon Sep 17 00:00:00 2001 From: apcraig Date: Sat, 9 Nov 2024 14:57:11 -0600 Subject: [PATCH] Add DoD ports to Carpenter, Narwhal, Warhawk. Limited testing but should work with machine set to "carpenter" and "narwhal". Add some pe layouts for ARRMwISC3to18 and ARRMwISC2p4to18 after scaling tests on Narwhal and Carpenter for the 3to18 configuration. More tuning probably needed. --- cime_config/allactive/config_pesall.xml | 4 +- .../cmake_macros/intel_carpenter.cmake | 14 + .../machines/cmake_macros/intel_narwhal.cmake | 10 + cime_config/machines/config_batch.xml | 69 ++++ cime_config/machines/config_machines.xml | 354 ++++++++++++++++-- cime_config/machines/config_pio.xml | 3 + .../mpas-ocean/cime_config/config_pes.xml | 117 +++++- 7 files changed, 545 insertions(+), 26 deletions(-) create mode 100644 cime_config/machines/cmake_macros/intel_carpenter.cmake diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 0d5f24565966..4c8ccfbcb0ff 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -2259,7 +2259,7 @@ - + none @@ -2296,7 +2296,7 @@ - + none diff --git a/cime_config/machines/cmake_macros/intel_carpenter.cmake b/cime_config/machines/cmake_macros/intel_carpenter.cmake new file mode 100644 index 000000000000..3e5477bb7b63 --- /dev/null +++ b/cime_config/machines/cmake_macros/intel_carpenter.cmake @@ -0,0 +1,14 @@ +# fp-model source does not work on in mpicxx, this seems to be a compiler bug, need to manually switch to precise +set(CMAKE_CXX_FLAGS " ") # hardcode it here to blank, then try to do same things as in intel.cmake +if (compile_threaded) + string(APPEND CMAKE_CXX_FLAGS " -qopenmp") +endif() +string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_CXX_FLAGS " -fp-model=precise") # and manually add precise +#message(STATUS "ndk CXXFLAGS=${CXXFLAGS}") + +string(APPEND CMAKE_Fortran_FLAGS " -fp-model consistent -fimf-use-svml") +string(APPEND CMAKE_CXX_FLAGS " -fp-model consistent") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -qno-opt-dynamic-align") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -lpthread") diff --git a/cime_config/machines/cmake_macros/intel_narwhal.cmake b/cime_config/machines/cmake_macros/intel_narwhal.cmake index 0981ed709a75..3e5477bb7b63 100644 --- a/cime_config/machines/cmake_macros/intel_narwhal.cmake +++ b/cime_config/machines/cmake_macros/intel_narwhal.cmake @@ -1,3 +1,13 @@ +# fp-model source does not work on in mpicxx, this seems to be a compiler bug, need to manually switch to precise +set(CMAKE_CXX_FLAGS " ") # hardcode it here to blank, then try to do same things as in intel.cmake +if (compile_threaded) + string(APPEND CMAKE_CXX_FLAGS " -qopenmp") +endif() +string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_CXX_FLAGS " -fp-model=precise") # and manually add precise +#message(STATUS "ndk CXXFLAGS=${CXXFLAGS}") + string(APPEND CMAKE_Fortran_FLAGS " -fp-model consistent -fimf-use-svml") string(APPEND CMAKE_CXX_FLAGS " -fp-model consistent") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -qno-opt-dynamic-align") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index ee271d1f9dc0..15cb6f2f9182 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -882,6 +882,29 @@ + + + -A {{ PROJECT }} + -l application=Regional-Arctic-System-Model + -l select={{ num_nodes }}:ncpus={{ MAX_MPITASKS_PER_NODE }}:mpiprocs={{ tasks_per_node }} + + + -A {{ PROJECT }} + -l application=Regional-Arctic-System-Model + -l select={{ num_nodes }}:ncpus={{ MAX_MPITASKS_PER_NODE }}:mpiprocs={{ tasks_per_node }} + + + -A {{ PROJECT }} + -l application=Regional-Arctic-System-Model + -l select=1:ncpus=1 + + + debug + standard + transfer + + + -A {{ PROJECT }} @@ -928,4 +951,50 @@ + + + -A {{ PROJECT }} + -l application=Regional-Arctic-System-Model + -l select={{ num_nodes }}:ncpus={{ MAX_MPITASKS_PER_NODE }}:mpiprocs={{ tasks_per_node }} + + + -A {{ PROJECT }} + -l application=Regional-Arctic-System-Model + -l select={{ num_nodes }}:ncpus={{ MAX_MPITASKS_PER_NODE }}:mpiprocs={{ tasks_per_node }} + + + -A {{ PROJECT }} + -l application=Regional-Arctic-System-Model + -l select=1:ncpus=1 + + + debug + standard + transfer + + + + + + -A {{ PROJECT }} + -l application=Regional-Arctic-System-Model + -l select={{ num_nodes }}:ncpus={{ MAX_MPITASKS_PER_NODE }}:mpiprocs={{ tasks_per_node }} + + + -A {{ PROJECT }} + -l application=Regional-Arctic-System-Model + -l select={{ num_nodes }}:ncpus={{ MAX_MPITASKS_PER_NODE }}:mpiprocs={{ tasks_per_node }} + + + -A {{ PROJECT }} + -l application=Regional-Arctic-System-Model + -l select=1:ncpus=1 + + + debug + standard + transfer + + + diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 36381d781535..2f48c42225fb 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -5388,6 +5388,115 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' + + ERDC HPE Cray EX4000 AMD, 192 pes/node, batch system is PBS + carpenter + CNL + intel + mpich + NPSCA07935242 + /p/app/unsupported/RASM/acme + .* + $ENV{WORKDIR} + /p/app/unsupported/RASM/acme/inputdata + /p/app/unsupported/RASM/acme/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /p/app/unsupported/RASM/acme/baselines/$COMPILER + /p/app/unsupported/RASM/tools/cprnc/cprnc + 8 + e3sm_developer + pbs + rasm + 192 + 192 + + override + aprun + + -n {{ total_tasks }} + -N $SHELL{if [ `./xmlquery --value MAX_MPITASKS_PER_NODE` -gt `./xmlquery --value TOTAL_TASKS` ];then echo `./xmlquery --value TOTAL_TASKS`;else echo `./xmlquery --value MAX_MPITASKS_PER_NODE`;fi;} + --cc depth -d $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} -j $SHELL{if [ 64 -ge `./xmlquery --value MAX_TASKS_PER_NODE` ];then echo 1;else echo `./xmlquery --value MAX_TASKS_PER_NODE`/64|bc;fi;} + + + + /opt/cray/pe/modules/3.2.11.7/init/perl.pm + /opt/cray/pe/modules/3.2.11.7/init/python.py + /opt/cray/pe/modules/3.2.11.7/init/sh + /opt/cray/pe/modules/3.2.11.7/init/csh + /opt/cray/pe/modules/3.2.11.7/bin/modulecmd perl + /opt/cray/pe/modules/3.2.11.7/bin/modulecmd python + module + module + + PrgEnv-intel + PrgEnv-cray + PrgEnv-gnu + PrgEnv-pgi + intel + cce + gcc + cray-parallel-netcdf + cray-parallel-hdf5 + pmi + cray-libsci + cray-mpich2 + cray-mpich + cray-netcdf + cray-hdf5 + cray-netcdf-hdf5parallel + craype + papi + cray-petsc + cray-libsci + esmf + craype + + + + PrgEnv-intel/8.4.0 + intel + intel/2023.0.0 + cray-mpich + cray-mpich/8.1.26 + cray-hdf5 + cray-hdf5-parallel + cray-netcdf-hdf5parallel + cray-parallel-netcdf + netcdf + cray-netcdf/4.9.0.3 + cray-hdf5/1.12.2.3 + cray-parallel-netcdf/1.12.3.3 + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + + $ENV{NETCDF_DIR} + $ENV{PNETCDF_DIR} + + + 1 + 1 + hybrid + /opt/cray/pe/netcdf/4.9.0.3/INTEL/19.0 + + + 64M + spread + threads + + + FALSE + + + yes + + + yes + + + NavyDSRC Cray EX, os is CNL, 128 pes/node, batch system is PBS narwhal @@ -5419,12 +5528,12 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /opt/cray/pe/modules/3.2.11.5/init/perl.pm - /opt/cray/pe/modules/3.2.11.5/init/python.py - /opt/cray/pe/modules/3.2.11.5/init/sh - /opt/cray/pe/modules/3.2.11.5/init/csh - /opt/cray/pe/modules/3.2.11.5/bin/modulecmd perl - /opt/cray/pe/modules/3.2.11.5/bin/modulecmd python + /opt/cray/pe/modules/3.2.11.7/init/perl.pm + /opt/cray/pe/modules/3.2.11.7/init/python.py + /opt/cray/pe/modules/3.2.11.7/init/sh + /opt/cray/pe/modules/3.2.11.7/init/csh + /opt/cray/pe/modules/3.2.11.7/bin/modulecmd perl + /opt/cray/pe/modules/3.2.11.7/bin/modulecmd python module module @@ -5443,25 +5552,33 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - PrgEnv-intel/8.0.0 + PrgEnv-intel intel intel-classic - intel-classic/2021.3.0 + intel-classic/2023.0.0 cray-mpich - cray-mpich/8.1.14 - cray-netcdf/4.7.4.7 - cray-hdf5/1.12.0.7 - cray-parallel-netcdf/1.12.1.7 + cray-mpich/8.1.26 + cray-pals + cray-pals/1.2.12 + cray-netcdf/4.9.0.1 + cray-hdf5/1.12.2.1 + cray-parallel-netcdf/1.12.3.1 cray-libsci - cray-libsci/21.08.1.2 + cray-libsci/22.08.1.1 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + $ENV{NETCDF_DIR} + $ENV{PNETCDF_DIR} + + + 1 1 + hybrid @@ -5540,18 +5657,213 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - PrgEnv-intel/8.0.0 + hpcx-ompi-intel + cray-netcdf/4.9.0.3 + cray-hdf5/1.12.2.3 + /app/DAAC/modules_internal + cmake/3.21.4 + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + + 1 + 1 + + + + 64M + spread + threads + + + FALSE + + + yes + + + yes + + + + + AFRL Cray EX, os is CNL, 128 pes/node, batch system is PBS + warhawk + CNL + intel + mpich + NPSCA07935242 + /p/work1/projects/RASM/acme + .* + $ENV{WORKDIR} + /p/work1/projects/RASM/acme/inputdata + /p/work1/projects/RASM/acme/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /p/work1/projects/RASM/acme/baselines/$COMPILER + /p/work1/projects/RASM/tools/cprnc/cprnc + 8 + e3sm_developer + pbs + rasm + 128 + 128 + + default + aprun + + -n {{ total_tasks }} + -N $SHELL{if [ `./xmlquery --value MAX_MPITASKS_PER_NODE` -gt `./xmlquery --value TOTAL_TASKS` ];then echo `./xmlquery --value TOTAL_TASKS`;else echo `./xmlquery --value MAX_MPITASKS_PER_NODE`;fi;} + --cc depth -d $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} -j $SHELL{if [ 64 -ge `./xmlquery --value MAX_TASKS_PER_NODE` ];then echo 1;else echo `./xmlquery --value MAX_TASKS_PER_NODE`/64|bc;fi;} + + + + /p/app/Modules/4.7.1/init/perl.pm + /p/app/Modules/4.7.1/init/python.py + /p/app/Modules/4.7.1/init/sh + /p/app/Modules/4.7.1/init/csh + /p/app/Modules/4.7.1/bin/modulecmd perl + /p/app/Modules/4.7.1/bin/modulecmd python + module + module + + PrgEnv-intel + PrgEnv-cray + PrgEnv-gnu + PrgEnv-nvidia + craype-x86-rome + craype-network-ofi + cray-dsmml + perftools-base + cray-libsci + cce + intel + gcc + cray-mpich + cray-hdf5 + cray-hdf5-parallel + cray-netcdf + cray-netcdf-hdf5parallel + cray-parallel-netcdf + + + + PrgEnv-intel/8.3.3 intel - intel-classic/2021.3.0 + intel-classic/2023.0.0 cray-mpich - cray-mpich/8.1.9 - cray-pals/1.0.17 - cray-netcdf/4.7.4.4 - cray-hdf5/1.12.0.4 - cray-parallel-netcdf/1.12.1.4 + cray-mpich/8.1.25 + cray-pals/1.2.11 + cray-netcdf/4.9.0.3 + cray-hdf5/1.12.2.3 + cray-parallel-netcdf/1.12.3.3 + /app/DAAC/modules_internal + cmake/3.21.4 + cray-libsci + cray-libsci/23.02.1.1 + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + + 1 + 1 + + + + 64M + spread + threads + + + FALSE + + + yes + + + yes + + + + + AFRL Cray EX, os is CNL, 128 pes/node, batch system is PBS + warhawk + CNL + intel + mpich + NPSCA07935242 + /p/work1/projects/RASM/acme + .* + $ENV{WORKDIR} + /p/work1/projects/RASM/acme/inputdata + /p/work1/projects/RASM/acme/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /p/work1/projects/RASM/acme/baselines/$COMPILER + /p/work1/projects/RASM/tools/cprnc/cprnc + 8 + e3sm_developer + pbs + rasm + 128 + 128 + + override + aprun + + -n {{ total_tasks }} + -N $SHELL{if [ `./xmlquery --value MAX_MPITASKS_PER_NODE` -gt `./xmlquery --value TOTAL_TASKS` ];then echo `./xmlquery --value TOTAL_TASKS`;else echo `./xmlquery --value MAX_MPITASKS_PER_NODE`;fi;} + --cc depth -d $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} -j $SHELL{if [ 64 -ge `./xmlquery --value MAX_TASKS_PER_NODE` ];then echo 1;else echo `./xmlquery --value MAX_TASKS_PER_NODE`/64|bc;fi;} + + + + /p/app/Modules/4.7.1/init/perl.pm + /p/app/Modules/4.7.1/init/python.py + /p/app/Modules/4.7.1/init/sh + /p/app/Modules/4.7.1/init/csh + /p/app/Modules/4.7.1/bin/modulecmd perl + /p/app/Modules/4.7.1/bin/modulecmd python + module + module + + PrgEnv-intel + PrgEnv-cray + PrgEnv-gnu + PrgEnv-nvidia + craype-x86-rome + craype-network-ofi + cray-dsmml + perftools-base cray-libsci + cce + intel + gcc + cray-mpich + cray-mpich-ucx + cray-hdf5 + cray-hdf5-parallel + cray-netcdf + cray-netcdf-hdf5parallel + cray-parallel-netcdf + + + + PrgEnv-intel/8.3.3 + intel + intel-classic/2023.0.0 + cray-mpich + cray-mpich-ucx + cray-mpich-ucx/8.1.25 + cray-pals/1.2.11 + cray-netcdf/4.9.0.3 + cray-hdf5/1.12.2.3 + cray-parallel-netcdf/1.12.3.3 + /app/DAAC/modules_internal cmake/3.21.4 - cray-libsci/21.08.1.2 + cray-libsci + cray-libsci/23.02.1.1 $CIME_OUTPUT_ROOT/$CASE/run diff --git a/cime_config/machines/config_pio.xml b/cime_config/machines/config_pio.xml index 742e82bd7ef7..08e12d406987 100644 --- a/cime_config/machines/config_pio.xml +++ b/cime_config/machines/config_pio.xml @@ -279,8 +279,11 @@ netcdf + netcdf netcdf netcdf + netcdf + netcdf diff --git a/components/mpas-ocean/cime_config/config_pes.xml b/components/mpas-ocean/cime_config/config_pes.xml index bc245761b85b..b5d3bba9ff0a 100644 --- a/components/mpas-ocean/cime_config/config_pes.xml +++ b/components/mpas-ocean/cime_config/config_pes.xml @@ -469,7 +469,7 @@ - + none @@ -506,7 +506,7 @@ - + none @@ -543,7 +543,7 @@ - + none @@ -579,4 +579,115 @@ + + + + none + + 128 + 1 + 128 + 3072 + 4096 + 1 + 1 + 3072 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 3072 + 0 + 0 + 0 + + + + + + + + none + + 192 + 1 + 192 + 5120 + 6000 + 1 + 1 + 5120 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 5120 + 0 + 0 + 0 + + + + + + + + none + + 128 + 1 + 128 + 2048 + 2048 + 1 + 1 + 2048 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 2048 + 0 + 0 + 0 + + + +