From ce98c5a075a3ebf4293fb923ce8ea3bc5962047b Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 11 Oct 2024 17:46:28 -0600 Subject: [PATCH 1/7] Update chicoma-cpu modules --- cime_config/machines/config_machines.xml | 50 ++++++++++++++---------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index acd10f74110b..b882536897ea 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4003,11 +4003,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4019,26 +4019,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 + PrgEnv-gnu/8.5.0 gcc/12.2.0 cray-libsci/23.05.1.4 @@ -4050,8 +4051,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 @@ -4060,15 +4061,22 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-libsci/23.05.1.4 - + + craype-accel-host + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 + + + craype-accel-host craype/2.7.21 cray-mpich/8.1.26 - libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 @@ -4090,6 +4098,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + -1 @@ -4131,11 +4142,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4202,7 +4213,6 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 From 457c9439ad86c41a8ef80db04b086e3d8957983c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 14 Oct 2024 14:43:58 -0600 Subject: [PATCH 2/7] Add latest cmake --- cime_config/machines/config_machines.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index b882536897ea..3c20372aa53b 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4068,6 +4068,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 + cmake/3.27.7 @@ -4077,6 +4078,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + cmake/3.27.7 @@ -4213,6 +4215,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + cmake/3.27.7 From a44f71e8c87618bd43961529feb6378306ca5a26 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 17 Oct 2024 09:34:47 -0600 Subject: [PATCH 3/7] Updates to get E3SM working on chicoma-cpu --- cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake | 1 + cime_config/machines/config_machines.xml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake index a6148451eb76..a6c13942620f 100644 --- a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake @@ -5,6 +5,7 @@ endif() set(PIO_FILESYSTEM_HINTS "lustre") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--enable-new-dtags") set(MPICC "cc") set(MPICXX "CC") set(MPIFC "ftn") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 3c20372aa53b..f893663f0ad8 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3983,7 +3983,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -4068,7 +4068,6 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - cmake/3.27.7 @@ -4078,6 +4077,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + + + cmake/3.27.7 From 699dea0bf09e638ee9a897381e3528cee44821a0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 19 Oct 2024 15:07:53 -0600 Subject: [PATCH 4/7] Update nvidia modules --- cime_config/machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index f893663f0ad8..941bcef51707 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4045,8 +4045,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - PrgEnv-nvidia/8.4.0 - nvidia/22.7 + PrgEnv-nvidia/8.5.0 + nvidia/24.7 cray-libsci/23.05.1.4 From 6453565d37e8bc24930c753fa5b9ff4867b42f85 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 18:31:01 -0600 Subject: [PATCH 5/7] Add GNU_CRAY_LDFLAGS flag to MPAS standalone builds This should be set to: ``` export GNU_CRAY_LDFLAGS="-Wl,--enable-new-dtags" ``` on Chicoma-CPU with gcc. --- components/mpas-framework/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 1f9b00844e72..f92606987bbb 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -396,11 +396,11 @@ gnu-cray: "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ - "LDFLAGS_OPT = -O3 -m64" \ + "LDFLAGS_OPT = -O3 -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -g -m64" \ - "LDFLAGS_DEBUG = -g -m64" \ + "LDFLAGS_DEBUG = -g -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_OMP = -fopenmp" \ "CFLAGS_OMP = -fopenmp" \ "BUILD_TARGET = $(@)" \ From 750707a82541eb245db50da51f0a686afcd14536 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 21:43:12 -0600 Subject: [PATCH 6/7] Revert chicoma-gpu changes (for now) --- cime_config/machines/config_machines.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 941bcef51707..3964b1405a2a 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4146,11 +4146,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/lmod/init/perl + /usr/share/lmod/8.3.1/init/perl - /usr/share/lmod/lmod/init/python - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh + /usr/share/lmod/8.3.1/init/python + /usr/share/lmod/8.3.1/init/sh + /usr/share/lmod/8.3.1/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4217,7 +4217,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.27.7 + cmake/3.25.1 From 0cb78eaff39e5290982f59e9eaf840f26de6a3c3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 21 Oct 2024 11:44:41 -0600 Subject: [PATCH 7/7] Update gcc and nvidia to newer versions This is to match proposed updates to Perlmutter-CPU https://github.com/E3SM-Project/E3SM/pull/6702 --- cime_config/machines/config_machines.xml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 3964b1405a2a..b1df10a96e11 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4019,6 +4019,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 + gcc + gcc-native intel intel-oneapi nvidia @@ -4040,14 +4042,14 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss PrgEnv-gnu/8.5.0 - gcc/12.2.0 - cray-libsci/23.05.1.4 + gcc-native/12.3 + cray-libsci/23.12.5 PrgEnv-nvidia/8.5.0 nvidia/24.7 - cray-libsci/23.05.1.4 + cray-libsci/23.12.5 @@ -4061,25 +4063,13 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-libsci/23.05.1.4 - + craype-accel-host craype/2.7.30 cray-mpich/8.1.28 cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - - - - craype-accel-host - craype/2.7.21 - cray-mpich/8.1.26 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - - - cmake/3.27.7 @@ -4103,7 +4093,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_PARALLEL_NETCDF_PREFIX} - /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + /usr/lib64/gcc/x86_64-suse-linux/12:$ENV{LD_LIBRARY_PATH} -1