From e11a718704aa804672e46842e942b34710263772 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Fri, 18 Oct 2024 12:20:05 -0500 Subject: [PATCH 01/13] Update `chicoma-cpu_gnu_mpich` files --- mache/spack/chicoma-cpu_gnu_mpich.csh | 13 +++++++------ mache/spack/chicoma-cpu_gnu_mpich.sh | 13 +++++++------ mache/spack/chicoma-cpu_gnu_mpich.yaml | 18 +++++++----------- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.csh b/mache/spack/chicoma-cpu_gnu_mpich.csh index 473312c1..98a30a0e 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.csh +++ b/mache/spack/chicoma-cpu_gnu_mpich.csh @@ -5,23 +5,23 @@ setenv HTTP_PROXY http://proxyout.lanl.gov:8080 setenv HTTPS_PROXY http://proxyout.lanl.gov:8080 setenv FTP_PROXY http://proxyout.lanl.gov:8080 -source /usr/share/lmod/8.3.1/init/csh +source /usr/share/lmod/lmod/init/csh +module rm craype-accel-nvidia80 +module rm craype-accel-host +module rm cray-libsci module rm PrgEnv-gnu module rm PrgEnv-nvidia module rm PrgEnv-cray module rm PrgEnv-aocc -module rm craype-accel-nvidia80 -module rm craype-accel-host -module load PrgEnv-gnu/8.4.0 +module load PrgEnv-gnu/8.5.0 module load gcc/12.2.0 module load craype-accel-host {% if e3sm_lapack %} module load cray-libsci {% endif %} module load craype -module load libfabric/1.15.2.0 module load cray-mpich/8.1.26 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel @@ -31,6 +31,7 @@ module load cray-hdf5-parallel/1.12.2.3 module load cray-netcdf-hdf5parallel/4.9.0.3 module load cray-parallel-netcdf/1.12.3.3 {% endif %} +module load cmake/3.27.7 setenv MPICH_ENV_DISPLAY 1 setenv MPICH_VERSION_DISPLAY 1 @@ -44,4 +45,4 @@ setenv PNETCDF_HINTS "romio_ds_write=disable;romio_ds_read=disable;romio_cb_writ setenv FI_CXI_RX_MATCH_MODE software setenv MPICH_COLL_SYNC MPI_Bcast -setenv LD_LIBRARY_PATH $CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH +setenv LD_LIBRARY_PATH="/opt/cray/pe/gcc/12.2.0/snos/lib64:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" diff --git a/mache/spack/chicoma-cpu_gnu_mpich.sh b/mache/spack/chicoma-cpu_gnu_mpich.sh index bd126e36..35543e1a 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.sh +++ b/mache/spack/chicoma-cpu_gnu_mpich.sh @@ -5,23 +5,23 @@ export HTTP_PROXY=http://proxyout.lanl.gov:8080 export HTTPS_PROXY=http://proxyout.lanl.gov:8080 export FTP_PROXY=http://proxyout.lanl.gov:8080 -source /usr/share/lmod/8.3.1/init/sh +source /usr/share/lmod/lmod/init/sh +module rm craype-accel-nvidia80 +module rm craype-accel-host +module rm cray-libsci module rm PrgEnv-gnu module rm PrgEnv-nvidia module rm PrgEnv-cray module rm PrgEnv-aocc -module rm craype-accel-nvidia80 -module rm craype-accel-host -module load PrgEnv-gnu/8.4.0 +module load PrgEnv-gnu/8.5.0 module load gcc/12.2.0 module load craype-accel-host {% if e3sm_lapack %} module load cray-libsci {% endif %} module load craype -module load libfabric/1.15.2.0 module load cray-mpich/8.1.26 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel @@ -31,6 +31,7 @@ module load cray-hdf5-parallel/1.12.2.3 module load cray-netcdf-hdf5parallel/4.9.0.3 module load cray-parallel-netcdf/1.12.3.3 {% endif %} +module load cmake/3.27.7 export MPICH_ENV_DISPLAY=1 export MPICH_VERSION_DISPLAY=1 @@ -44,4 +45,4 @@ export PNETCDF_HINTS="romio_ds_write=disable;romio_ds_read=disable;romio_cb_writ export FI_CXI_RX_MATCH_MODE=software export MPICH_COLL_SYNC=MPI_Bcast -export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH="/opt/cray/pe/gcc/12.2.0/snos/lib64:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" diff --git a/mache/spack/chicoma-cpu_gnu_mpich.yaml b/mache/spack/chicoma-cpu_gnu_mpich.yaml index a31c3ad9..faa27d91 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.yaml +++ b/mache/spack/chicoma-cpu_gnu_mpich.yaml @@ -27,6 +27,11 @@ spack: - spec: bzip2@1.0.6 prefix: /usr buildable: false + cmake: + externals: + - spec: cmake@3.27.7 + prefix: /usr/projects/hpcsoft/tce/23-05/cos2-x86_64-cc80/packages/cmake/cmake-3.27.7/ + buildable: false curl: externals: - spec: curl@7.60.0 @@ -93,27 +98,18 @@ spack: externals: - spec: gcc@12.2.0 modules: - - PrgEnv-gnu/8.4.0 + - PrgEnv-gnu/8.5.0 - gcc/12.2.0 - craype-accel-host - craype - - libfabric/1.15.2.0 buildable: false cray-mpich: externals: - spec: cray-mpich@8.1.26 prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/9.1 modules: - - libfabric/1.15.2.0 - cray-mpich/8.1.26 buildable: false - libfabric: - externals: - - spec: libfabric@1.15.2.0 - prefix: /opt/cray/libfabric/1.15.2.0 - modules: - - libfabric/1.15.2.0 - buildable: false {% if e3sm_lapack %} cray-libsci: externals: @@ -161,7 +157,7 @@ spack: operating_system: sles15 target: x86_64 modules: - - PrgEnv-gnu/8.4.0 + - PrgEnv-gnu/8.5.0 - gcc/12.2.0 - craype-accel-host - craype From cb8e34a6df1e044ad258aeca24665d7a6f9468e6 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Fri, 18 Oct 2024 12:34:28 -0500 Subject: [PATCH 02/13] Update `chicoma-cpu_nvidia_mpich` files --- mache/spack/chicoma-cpu_nvidia_mpich.csh | 9 +++++---- mache/spack/chicoma-cpu_nvidia_mpich.sh | 9 +++++---- mache/spack/chicoma-cpu_nvidia_mpich.yaml | 13 ++----------- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.csh b/mache/spack/chicoma-cpu_nvidia_mpich.csh index ff595fc7..72f933c1 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.csh +++ b/mache/spack/chicoma-cpu_nvidia_mpich.csh @@ -5,19 +5,19 @@ setenv HTTP_PROXY http://proxyout.lanl.gov:8080 setenv HTTPS_PROXY http://proxyout.lanl.gov:8080 setenv FTP_PROXY http://proxyout.lanl.gov:8080 -source /usr/share/lmod/8.3.1/init/csh +source /usr/share/lmod/lmod/init/csh +module rm craype-accel-nvidia80 +module rm craype-accel-host +module rm cray-libsci module rm PrgEnv-gnu module rm PrgEnv-nvidia module rm PrgEnv-cray module rm PrgEnv-aocc -module rm craype-accel-nvidia80 -module rm craype-accel-host module load PrgEnv-nvidia/8.4.0 module load nvidia/22.7 module load craype-x86-milan -module load libfabric/1.15.2.0 module load craype-accel-host module load craype module load cray-mpich/8.1.26 @@ -32,6 +32,7 @@ module load cray-hdf5-parallel/1.12.2.3 module load cray-netcdf-hdf5parallel/4.9.0.3 module load cray-parallel-netcdf/1.12.3.3 {% endif %} +module load cmake/3.27.7 setenv MPICH_ENV_DISPLAY 1 setenv MPICH_VERSION_DISPLAY 1 diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.sh b/mache/spack/chicoma-cpu_nvidia_mpich.sh index b0e6ed52..b7f2794f 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.sh +++ b/mache/spack/chicoma-cpu_nvidia_mpich.sh @@ -5,19 +5,19 @@ export HTTP_PROXY=http://proxyout.lanl.gov:8080 export HTTPS_PROXY=http://proxyout.lanl.gov:8080 export FTP_PROXY=http://proxyout.lanl.gov:8080 -source /usr/share/lmod/8.3.1/init/sh +source /usr/share/lmod/lmod/init/sh +module rm craype-accel-nvidia80 +module rm craype-accel-host +module rm cray-libsci module rm PrgEnv-gnu module rm PrgEnv-nvidia module rm PrgEnv-cray module rm PrgEnv-aocc -module rm craype-accel-nvidia80 -module rm craype-accel-host module load PrgEnv-nvidia/8.4.0 module load nvidia/22.7 module load craype-x86-milan -module load libfabric/1.15.2.0 module load craype-accel-host module load craype module load cray-mpich/8.1.26 @@ -32,6 +32,7 @@ module load cray-hdf5-parallel/1.12.2.3 module load cray-netcdf-hdf5parallel/4.9.0.3 module load cray-parallel-netcdf/1.12.3.3 {% endif %} +module load cmake/3.27.7 export MPICH_ENV_DISPLAY=1 export MPICH_VERSION_DISPLAY=1 diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.yaml b/mache/spack/chicoma-cpu_nvidia_mpich.yaml index 8f9be377..f9609515 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.yaml +++ b/mache/spack/chicoma-cpu_nvidia_mpich.yaml @@ -28,8 +28,8 @@ spack: buildable: false cmake: externals: - - spec: cmake@3.25.1 - prefix: /usr/projects/hpcsoft/cos2/common/x86_64/cmake/3.25.1 + - spec: cmake@3.27.7 + prefix: /usr/projects/hpcsoft/tce/23-05/cos2-x86_64-cc80/packages/cmake/cmake-3.27.7/ buildable: false curl: externals: @@ -98,16 +98,8 @@ spack: - spec: cray-mpich@8.1.26 prefix: /opt/cray/pe/mpich/8.1.26/ofi/nvidia/20.7 modules: - - libfabric/1.15.2.0 - cray-mpich/8.1.26 buildable: false - libfabric: - externals: - - spec: libfabric@1.15.2.0 - prefix: /opt/cray/libfabric/1.15.2.0 - modules: - - libfabric/1.15.2.0 - buildable: false {% if e3sm_lapack %} cray-libsci: externals: @@ -157,7 +149,6 @@ spack: - nvidia/22.7 - craype-x86-milan - craype-accel-host - - libfabric/1.15.2.0 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.5.2-2.4_3.45__gd0f7936.shasta/lib64/pkgconfig" From 8dfc697a831edd7135508b920f6f8b7fa784f863 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 19 Oct 2024 14:46:20 -0600 Subject: [PATCH 03/13] Fix module rm commands --- mache/spack/chicoma-cpu_gnu_mpich.csh | 20 ++++++++++++++++---- mache/spack/chicoma-cpu_gnu_mpich.sh | 20 ++++++++++++++++---- mache/spack/chicoma-cpu_nvidia_mpich.csh | 20 ++++++++++++++++---- mache/spack/chicoma-cpu_nvidia_mpich.sh | 20 ++++++++++++++++---- 4 files changed, 64 insertions(+), 16 deletions(-) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.csh b/mache/spack/chicoma-cpu_gnu_mpich.csh index 98a30a0e..78db5927 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.csh +++ b/mache/spack/chicoma-cpu_gnu_mpich.csh @@ -7,10 +7,25 @@ setenv FTP_PROXY http://proxyout.lanl.gov:8080 source /usr/share/lmod/lmod/init/csh +module rm cray-hdf5-parallel +module rm cray-netcdf-hdf5parallel +module rm cray-parallel-netcdf +module rm cray-netcdf +module rm cray-hdf5 +module rm intel +module rm intel-oneapi +module rm nvidia +module rm aocc +module rm cudatoolkit +module rm climate-utils +module rm cray-libsci module rm craype-accel-nvidia80 module rm craype-accel-host -module rm cray-libsci +module rm perftools-base +module rm perftools +module rm darshan module rm PrgEnv-gnu +module rm PrgEnv-intel module rm PrgEnv-nvidia module rm PrgEnv-cray module rm PrgEnv-aocc @@ -24,9 +39,6 @@ module load cray-libsci module load craype module load cray-mpich/8.1.26 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel -module rm cray-netcdf-hdf5parallel -module rm cray-parallel-netcdf module load cray-hdf5-parallel/1.12.2.3 module load cray-netcdf-hdf5parallel/4.9.0.3 module load cray-parallel-netcdf/1.12.3.3 diff --git a/mache/spack/chicoma-cpu_gnu_mpich.sh b/mache/spack/chicoma-cpu_gnu_mpich.sh index 35543e1a..7ed7616c 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.sh +++ b/mache/spack/chicoma-cpu_gnu_mpich.sh @@ -7,10 +7,25 @@ export FTP_PROXY=http://proxyout.lanl.gov:8080 source /usr/share/lmod/lmod/init/sh +module rm cray-hdf5-parallel +module rm cray-netcdf-hdf5parallel +module rm cray-parallel-netcdf +module rm cray-netcdf +module rm cray-hdf5 +module rm intel +module rm intel-oneapi +module rm nvidia +module rm aocc +module rm cudatoolkit +module rm climate-utils +module rm cray-libsci module rm craype-accel-nvidia80 module rm craype-accel-host -module rm cray-libsci +module rm perftools-base +module rm perftools +module rm darshan module rm PrgEnv-gnu +module rm PrgEnv-intel module rm PrgEnv-nvidia module rm PrgEnv-cray module rm PrgEnv-aocc @@ -24,9 +39,6 @@ module load cray-libsci module load craype module load cray-mpich/8.1.26 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel -module rm cray-netcdf-hdf5parallel -module rm cray-parallel-netcdf module load cray-hdf5-parallel/1.12.2.3 module load cray-netcdf-hdf5parallel/4.9.0.3 module load cray-parallel-netcdf/1.12.3.3 diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.csh b/mache/spack/chicoma-cpu_nvidia_mpich.csh index 72f933c1..e5cdc9f8 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.csh +++ b/mache/spack/chicoma-cpu_nvidia_mpich.csh @@ -7,10 +7,25 @@ setenv FTP_PROXY http://proxyout.lanl.gov:8080 source /usr/share/lmod/lmod/init/csh +module rm cray-hdf5-parallel +module rm cray-netcdf-hdf5parallel +module rm cray-parallel-netcdf +module rm cray-netcdf +module rm cray-hdf5 +module rm intel +module rm intel-oneapi +module rm nvidia +module rm aocc +module rm cudatoolkit +module rm climate-utils +module rm cray-libsci module rm craype-accel-nvidia80 module rm craype-accel-host -module rm cray-libsci +module rm perftools-base +module rm perftools +module rm darshan module rm PrgEnv-gnu +module rm PrgEnv-intel module rm PrgEnv-nvidia module rm PrgEnv-cray module rm PrgEnv-aocc @@ -25,9 +40,6 @@ module load cray-mpich/8.1.26 module load cray-libsci/23.05.1.4 {% endif %} {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel -module rm cray-netcdf-hdf5parallel -module rm cray-parallel-netcdf module load cray-hdf5-parallel/1.12.2.3 module load cray-netcdf-hdf5parallel/4.9.0.3 module load cray-parallel-netcdf/1.12.3.3 diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.sh b/mache/spack/chicoma-cpu_nvidia_mpich.sh index b7f2794f..e4e07174 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.sh +++ b/mache/spack/chicoma-cpu_nvidia_mpich.sh @@ -7,10 +7,25 @@ export FTP_PROXY=http://proxyout.lanl.gov:8080 source /usr/share/lmod/lmod/init/sh +module rm cray-hdf5-parallel +module rm cray-netcdf-hdf5parallel +module rm cray-parallel-netcdf +module rm cray-netcdf +module rm cray-hdf5 +module rm intel +module rm intel-oneapi +module rm nvidia +module rm aocc +module rm cudatoolkit +module rm climate-utils +module rm cray-libsci module rm craype-accel-nvidia80 module rm craype-accel-host -module rm cray-libsci +module rm perftools-base +module rm perftools +module rm darshan module rm PrgEnv-gnu +module rm PrgEnv-intel module rm PrgEnv-nvidia module rm PrgEnv-cray module rm PrgEnv-aocc @@ -25,9 +40,6 @@ module load cray-mpich/8.1.26 module load cray-libsci/23.05.1.4 {% endif %} {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel -module rm cray-netcdf-hdf5parallel -module rm cray-parallel-netcdf module load cray-hdf5-parallel/1.12.2.3 module load cray-netcdf-hdf5parallel/4.9.0.3 module load cray-parallel-netcdf/1.12.3.3 From d38943ccc9e029c92ba4a2bf6168f8be9c9388a7 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 19 Oct 2024 15:21:34 -0600 Subject: [PATCH 04/13] Remove and load modules together At least to the extent possible. --- mache/spack/chicoma-cpu_gnu_mpich.csh | 65 ++++++++++++++------------- mache/spack/chicoma-cpu_gnu_mpich.sh | 65 ++++++++++++++------------- 2 files changed, 66 insertions(+), 64 deletions(-) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.csh b/mache/spack/chicoma-cpu_gnu_mpich.csh index 78db5927..3c36ba82 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.csh +++ b/mache/spack/chicoma-cpu_gnu_mpich.csh @@ -7,43 +7,44 @@ setenv FTP_PROXY http://proxyout.lanl.gov:8080 source /usr/share/lmod/lmod/init/csh -module rm cray-hdf5-parallel -module rm cray-netcdf-hdf5parallel -module rm cray-parallel-netcdf -module rm cray-netcdf -module rm cray-hdf5 -module rm intel -module rm intel-oneapi -module rm nvidia -module rm aocc -module rm cudatoolkit -module rm climate-utils -module rm cray-libsci -module rm craype-accel-nvidia80 -module rm craype-accel-host -module rm perftools-base -module rm perftools -module rm darshan -module rm PrgEnv-gnu -module rm PrgEnv-intel -module rm PrgEnv-nvidia -module rm PrgEnv-cray -module rm PrgEnv-aocc +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + cray-netcdf \ + cray-hdf5 \ + intel \ + intel-oneapi \ + nvidia \ + aocc \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + craype \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc -module load PrgEnv-gnu/8.5.0 -module load gcc/12.2.0 -module load craype-accel-host +module load PrgEnv-gnu/8.5.0 \ + gcc/12.2.0 \ + craype-accel-host \ + cray-mpich/8.1.26 \ + craype \ + cmake/3.27.7 {% if e3sm_lapack %} -module load cray-libsci +module load cray-libsci/23.05.1.4 {% endif %} -module load craype -module load cray-mpich/8.1.26 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} -module load cmake/3.27.7 setenv MPICH_ENV_DISPLAY 1 setenv MPICH_VERSION_DISPLAY 1 diff --git a/mache/spack/chicoma-cpu_gnu_mpich.sh b/mache/spack/chicoma-cpu_gnu_mpich.sh index 7ed7616c..145de5e3 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.sh +++ b/mache/spack/chicoma-cpu_gnu_mpich.sh @@ -7,43 +7,44 @@ export FTP_PROXY=http://proxyout.lanl.gov:8080 source /usr/share/lmod/lmod/init/sh -module rm cray-hdf5-parallel -module rm cray-netcdf-hdf5parallel -module rm cray-parallel-netcdf -module rm cray-netcdf -module rm cray-hdf5 -module rm intel -module rm intel-oneapi -module rm nvidia -module rm aocc -module rm cudatoolkit -module rm climate-utils -module rm cray-libsci -module rm craype-accel-nvidia80 -module rm craype-accel-host -module rm perftools-base -module rm perftools -module rm darshan -module rm PrgEnv-gnu -module rm PrgEnv-intel -module rm PrgEnv-nvidia -module rm PrgEnv-cray -module rm PrgEnv-aocc +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + cray-netcdf \ + cray-hdf5 \ + intel \ + intel-oneapi \ + nvidia \ + aocc \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + craype \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc -module load PrgEnv-gnu/8.5.0 -module load gcc/12.2.0 -module load craype-accel-host +module load PrgEnv-gnu/8.5.0 \ + gcc/12.2.0 \ + craype-accel-host \ + cray-mpich/8.1.26 \ + craype \ + cmake/3.27.7 {% if e3sm_lapack %} -module load cray-libsci +module load cray-libsci/23.05.1.4 {% endif %} -module load craype -module load cray-mpich/8.1.26 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} -module load cmake/3.27.7 export MPICH_ENV_DISPLAY=1 export MPICH_VERSION_DISPLAY=1 From 0b6e718316d5d0c13efd18cd409c17e511ec5cce Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 19 Oct 2024 21:59:10 -0600 Subject: [PATCH 05/13] Remove libfabric module --- mache/spack/chicoma-cpu_gnu_mpich.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.yaml b/mache/spack/chicoma-cpu_gnu_mpich.yaml index faa27d91..0c6f4b24 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.yaml +++ b/mache/spack/chicoma-cpu_gnu_mpich.yaml @@ -161,7 +161,6 @@ spack: - gcc/12.2.0 - craype-accel-host - craype - - libfabric/1.15.2.0 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.5.2-2.4_3.45__gd0f7936.shasta/lib64/pkgconfig" From 57fb07ebeb10c3e620f9ddcefe5e0ef983f15e70 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 19 Oct 2024 23:25:55 -0600 Subject: [PATCH 06/13] Use modules rather than prefixes for several packages --- mache/spack/chicoma-cpu_gnu_mpich.yaml | 3 --- mache/spack/chicoma-cpu_nvidia_mpich.yaml | 5 ++--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.yaml b/mache/spack/chicoma-cpu_gnu_mpich.yaml index 0c6f4b24..56d3aa02 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.yaml +++ b/mache/spack/chicoma-cpu_gnu_mpich.yaml @@ -106,7 +106,6 @@ spack: cray-mpich: externals: - spec: cray-mpich@8.1.26 - prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/9.1 modules: - cray-mpich/8.1.26 buildable: false @@ -114,7 +113,6 @@ spack: cray-libsci: externals: - spec: cray-libsci@23.05.1.4 - prefix: /opt/cray/pe/libsci/23.05.1.4/GNU/9.1/x86_64 modules: - cray-libsci/23.05.1.4 buildable: false @@ -123,7 +121,6 @@ spack: hdf5: externals: - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1 modules: - cray-hdf5-parallel/1.12.2.3 buildable: false diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.yaml b/mache/spack/chicoma-cpu_nvidia_mpich.yaml index f9609515..7fac0b74 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.yaml +++ b/mache/spack/chicoma-cpu_nvidia_mpich.yaml @@ -96,7 +96,6 @@ spack: cray-mpich: externals: - spec: cray-mpich@8.1.26 - prefix: /opt/cray/pe/mpich/8.1.26/ofi/nvidia/20.7 modules: - cray-mpich/8.1.26 buildable: false @@ -104,14 +103,14 @@ spack: cray-libsci: externals: - spec: cray-libsci@23.05.1.4 - prefix: /opt/cray/pe/libsci/23.05.1.4/NVIDIA/20.7/x86_64 + modules: + - cray-libsci/23.05.1.4 buildable: false {% endif %} {% if e3sm_hdf5_netcdf %} hdf5: externals: - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/nvidia/20.7 modules: - cray-hdf5-parallel/1.12.2.3 buildable: false From 9a36f4ed55cbf7d11a0a20f5e0b838d684cc4cd4 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 10:50:02 -0600 Subject: [PATCH 07/13] Require cray-libsci as part of compiler stack gcc fails without it --- mache/spack/chicoma-cpu_gnu_mpich.csh | 5 ++--- mache/spack/chicoma-cpu_gnu_mpich.sh | 5 ++--- mache/spack/chicoma-cpu_gnu_mpich.yaml | 8 ++------ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.csh b/mache/spack/chicoma-cpu_gnu_mpich.csh index 3c36ba82..b481e2aa 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.csh +++ b/mache/spack/chicoma-cpu_gnu_mpich.csh @@ -31,15 +31,14 @@ module rm cray-hdf5-parallel \ PrgEnv-cray \ PrgEnv-aocc +# we must load cray-libsci for gcc to work module load PrgEnv-gnu/8.5.0 \ gcc/12.2.0 \ + cray-libsci/23.05.1.4 \ craype-accel-host \ cray-mpich/8.1.26 \ craype \ cmake/3.27.7 -{% if e3sm_lapack %} -module load cray-libsci/23.05.1.4 -{% endif %} {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.3 \ cray-netcdf-hdf5parallel/4.9.0.3 \ diff --git a/mache/spack/chicoma-cpu_gnu_mpich.sh b/mache/spack/chicoma-cpu_gnu_mpich.sh index 145de5e3..e5e0e1fd 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.sh +++ b/mache/spack/chicoma-cpu_gnu_mpich.sh @@ -31,15 +31,14 @@ module rm cray-hdf5-parallel \ PrgEnv-cray \ PrgEnv-aocc +# we must load cray-libsci for gcc to work module load PrgEnv-gnu/8.5.0 \ gcc/12.2.0 \ + cray-libsci/23.05.1.4 \ craype-accel-host \ cray-mpich/8.1.26 \ craype \ cmake/3.27.7 -{% if e3sm_lapack %} -module load cray-libsci/23.05.1.4 -{% endif %} {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.3 \ cray-netcdf-hdf5parallel/4.9.0.3 \ diff --git a/mache/spack/chicoma-cpu_gnu_mpich.yaml b/mache/spack/chicoma-cpu_gnu_mpich.yaml index 56d3aa02..80ecff4b 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.yaml +++ b/mache/spack/chicoma-cpu_gnu_mpich.yaml @@ -2,9 +2,7 @@ spack: specs: - gcc - cray-mpich -{% if e3sm_lapack %} - cray-libsci -{% endif %} {% if e3sm_hdf5_netcdf %} - hdf5 - netcdf-c @@ -19,9 +17,7 @@ spack: compiler: [gcc@12.2.0] providers: mpi: [cray-mpich@8.1.26] -{% if e3sm_lapack %} lapack: [cray-libsci@23.05.1.4] -{% endif %} bzip2: externals: - spec: bzip2@1.0.6 @@ -100,6 +96,7 @@ spack: modules: - PrgEnv-gnu/8.5.0 - gcc/12.2.0 + - cray-libsci/23.05.1.4 - craype-accel-host - craype buildable: false @@ -109,14 +106,12 @@ spack: modules: - cray-mpich/8.1.26 buildable: false -{% if e3sm_lapack %} cray-libsci: externals: - spec: cray-libsci@23.05.1.4 modules: - cray-libsci/23.05.1.4 buildable: false -{% endif %} {% if e3sm_hdf5_netcdf %} hdf5: externals: @@ -156,6 +151,7 @@ spack: modules: - PrgEnv-gnu/8.5.0 - gcc/12.2.0 + - cray-libsci/23.05.1.4 - craype-accel-host - craype environment: From 40f21cf96221929e7727296dc7dc4cbd825bc0a1 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 11:28:51 -0600 Subject: [PATCH 08/13] Add libfabric (back) and craype-x86-rome --- mache/spack/chicoma-cpu_gnu_mpich.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.yaml b/mache/spack/chicoma-cpu_gnu_mpich.yaml index 80ecff4b..26d4b1fd 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.yaml +++ b/mache/spack/chicoma-cpu_gnu_mpich.yaml @@ -96,9 +96,11 @@ spack: modules: - PrgEnv-gnu/8.5.0 - gcc/12.2.0 + - libfabric/1.15.2.0 - cray-libsci/23.05.1.4 - - craype-accel-host - craype + - craype-accel-host + - craype-x86-rome buildable: false cray-mpich: externals: @@ -151,9 +153,11 @@ spack: modules: - PrgEnv-gnu/8.5.0 - gcc/12.2.0 + - libfabric/1.15.2.0 - cray-libsci/23.05.1.4 - - craype-accel-host - craype + - craype-accel-host + - craype-x86-rome environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.5.2-2.4_3.45__gd0f7936.shasta/lib64/pkgconfig" From 71768aede2d9c4e4d623d7cb9fa8d241276177b9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 14:28:45 -0600 Subject: [PATCH 09/13] Switch hdf5 back to prefix, not module --- mache/spack/chicoma-cpu_gnu_mpich.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.yaml b/mache/spack/chicoma-cpu_gnu_mpich.yaml index 26d4b1fd..b50ef62c 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.yaml +++ b/mache/spack/chicoma-cpu_gnu_mpich.yaml @@ -118,8 +118,7 @@ spack: hdf5: externals: - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared - modules: - - cray-hdf5-parallel/1.12.2.3 + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/gnu/9.1 buildable: false parallel-netcdf: externals: From 7618650d9d6b48bcd6b321a2fb0cb0e4e541b6a6 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 15:11:51 -0600 Subject: [PATCH 10/13] Update nvidia on chicoma-cpu --- mache/spack/chicoma-cpu_nvidia_mpich.csh | 69 +++++++++++------------ mache/spack/chicoma-cpu_nvidia_mpich.sh | 68 +++++++++++----------- mache/spack/chicoma-cpu_nvidia_mpich.yaml | 27 ++++----- 3 files changed, 79 insertions(+), 85 deletions(-) diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.csh b/mache/spack/chicoma-cpu_nvidia_mpich.csh index e5cdc9f8..c19c29d2 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.csh +++ b/mache/spack/chicoma-cpu_nvidia_mpich.csh @@ -7,44 +7,43 @@ setenv FTP_PROXY http://proxyout.lanl.gov:8080 source /usr/share/lmod/lmod/init/csh -module rm cray-hdf5-parallel -module rm cray-netcdf-hdf5parallel -module rm cray-parallel-netcdf -module rm cray-netcdf -module rm cray-hdf5 -module rm intel -module rm intel-oneapi -module rm nvidia -module rm aocc -module rm cudatoolkit -module rm climate-utils -module rm cray-libsci -module rm craype-accel-nvidia80 -module rm craype-accel-host -module rm perftools-base -module rm perftools -module rm darshan -module rm PrgEnv-gnu -module rm PrgEnv-intel -module rm PrgEnv-nvidia -module rm PrgEnv-cray -module rm PrgEnv-aocc -module load PrgEnv-nvidia/8.4.0 -module load nvidia/22.7 -module load craype-x86-milan -module load craype-accel-host -module load craype -module load cray-mpich/8.1.26 -{% if e3sm_lapack %} -module load cray-libsci/23.05.1.4 -{% endif %} +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + cray-netcdf \ + cray-hdf5 \ + intel \ + intel-oneapi \ + nvidia \ + aocc \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + craype \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc + +module load PrgEnv-nvidia/8.5.0 \ + nvidia/24.7 \ + cray-libsci/23.05.1.4 \ + craype-accel-host \ + cray-mpich/8.1.26 \ + craype \ + cmake/3.27.7 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} -module load cmake/3.27.7 setenv MPICH_ENV_DISPLAY 1 setenv MPICH_VERSION_DISPLAY 1 diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.sh b/mache/spack/chicoma-cpu_nvidia_mpich.sh index e4e07174..bd2582c7 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.sh +++ b/mache/spack/chicoma-cpu_nvidia_mpich.sh @@ -7,44 +7,42 @@ export FTP_PROXY=http://proxyout.lanl.gov:8080 source /usr/share/lmod/lmod/init/sh -module rm cray-hdf5-parallel -module rm cray-netcdf-hdf5parallel -module rm cray-parallel-netcdf -module rm cray-netcdf -module rm cray-hdf5 -module rm intel -module rm intel-oneapi -module rm nvidia -module rm aocc -module rm cudatoolkit -module rm climate-utils -module rm cray-libsci -module rm craype-accel-nvidia80 -module rm craype-accel-host -module rm perftools-base -module rm perftools -module rm darshan -module rm PrgEnv-gnu -module rm PrgEnv-intel -module rm PrgEnv-nvidia -module rm PrgEnv-cray -module rm PrgEnv-aocc +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + cray-netcdf \ + cray-hdf5 \ + intel \ + intel-oneapi \ + nvidia \ + aocc \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + craype \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc -module load PrgEnv-nvidia/8.4.0 -module load nvidia/22.7 -module load craype-x86-milan -module load craype-accel-host -module load craype -module load cray-mpich/8.1.26 -{% if e3sm_lapack %} -module load cray-libsci/23.05.1.4 -{% endif %} +module load PrgEnv-nvidia/8.5.0 \ + nvidia/24.7 \ + cray-libsci/23.05.1.4 \ + craype-accel-host \ + cray-mpich/8.1.26 \ + craype \ + cmake/3.27.7 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} -module load cmake/3.27.7 export MPICH_ENV_DISPLAY=1 export MPICH_VERSION_DISPLAY=1 diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.yaml b/mache/spack/chicoma-cpu_nvidia_mpich.yaml index 7fac0b74..7cd2d934 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.yaml +++ b/mache/spack/chicoma-cpu_nvidia_mpich.yaml @@ -1,9 +1,7 @@ spack: specs: - cray-mpich -{% if e3sm_lapack %} - cray-libsci -{% endif %} {% if e3sm_hdf5_netcdf %} - hdf5 - netcdf-c @@ -15,12 +13,10 @@ spack: unify: true packages: all: - compiler: [nvhpc@22.7] + compiler: [nvhpc@24.7] providers: mpi: [cray-mpich@8.1.26] -{% if e3sm_lapack %} lapack: [cray-libsci@23.05.1.4] -{% endif %} bzip2: externals: - spec: bzip2@1.0.6 @@ -99,14 +95,12 @@ spack: modules: - cray-mpich/8.1.26 buildable: false -{% if e3sm_lapack %} cray-libsci: externals: - spec: cray-libsci@23.05.1.4 modules: - cray-libsci/23.05.1.4 buildable: false -{% endif %} {% if e3sm_hdf5_netcdf %} hdf5: externals: @@ -134,20 +128,23 @@ spack: install_missing_compilers: false compilers: - compiler: - spec: nvhpc@22.7 + spec: nvhpc@24.7 paths: - cc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc - cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc++ - f77: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran - fc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran + cc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.7/compilers/bin/nvc + cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/24.7/compilers/bin/nvc++ + f77: /opt/nvidia/hpc_sdk/Linux_x86_64/24.7/compilers/bin/nvfortran + fc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.7/compilers/bin/nvfortran flags: {} operating_system: sles15 target: any modules: - - PrgEnv-nvidia/8.4.0 - - nvidia/22.7 - - craype-x86-milan + - PrgEnv-nvidia/8.5.0 + - nvidia/24.7 + - libfabric/1.15.2.0 + - cray-libsci/23.05.1.4 + - craype - craype-accel-host + - craype-x86-rome environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.5.2-2.4_3.45__gd0f7936.shasta/lib64/pkgconfig" From d61152c7a2b62738750ed8f2ccfb9c391cd908d3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 18:36:40 -0600 Subject: [PATCH 11/13] Set GNU_CRAY_LDFLAGS for MPAS standalone builds --- mache/spack/chicoma-cpu_gnu_mpich.csh | 3 +++ mache/spack/chicoma-cpu_gnu_mpich.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.csh b/mache/spack/chicoma-cpu_gnu_mpich.csh index b481e2aa..3d88f3a4 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.csh +++ b/mache/spack/chicoma-cpu_gnu_mpich.csh @@ -57,4 +57,7 @@ setenv PNETCDF_HINTS "romio_ds_write=disable;romio_ds_read=disable;romio_cb_writ setenv FI_CXI_RX_MATCH_MODE software setenv MPICH_COLL_SYNC MPI_Bcast +# for standalone MPAS builds +setenv GNU_CRAY_LDFLAGS "-Wl,--enable-new-dtags" + setenv LD_LIBRARY_PATH="/opt/cray/pe/gcc/12.2.0/snos/lib64:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" diff --git a/mache/spack/chicoma-cpu_gnu_mpich.sh b/mache/spack/chicoma-cpu_gnu_mpich.sh index e5e0e1fd..27d64a5d 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.sh +++ b/mache/spack/chicoma-cpu_gnu_mpich.sh @@ -58,3 +58,6 @@ export FI_CXI_RX_MATCH_MODE=software export MPICH_COLL_SYNC=MPI_Bcast export LD_LIBRARY_PATH="/opt/cray/pe/gcc/12.2.0/snos/lib64:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" + +# for standalone MPAS builds +export GNU_CRAY_LDFLAGS="-Wl,--enable-new-dtags" From 168138c00a5ca86bcefd4ae59fd79b687cec709a Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 21 Oct 2024 14:00:49 -0600 Subject: [PATCH 12/13] Update to newer gnu modules --- mache/spack/chicoma-cpu_gnu_mpich.csh | 18 ++++++----- mache/spack/chicoma-cpu_gnu_mpich.sh | 18 ++++++----- mache/spack/chicoma-cpu_gnu_mpich.yaml | 44 +++++++++++++------------- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/mache/spack/chicoma-cpu_gnu_mpich.csh b/mache/spack/chicoma-cpu_gnu_mpich.csh index 3d88f3a4..434a23cd 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.csh +++ b/mache/spack/chicoma-cpu_gnu_mpich.csh @@ -12,6 +12,8 @@ module rm cray-hdf5-parallel \ cray-parallel-netcdf \ cray-netcdf \ cray-hdf5 \ + gcc \ + gcc-native \ intel \ intel-oneapi \ nvidia \ @@ -33,16 +35,16 @@ module rm cray-hdf5-parallel \ # we must load cray-libsci for gcc to work module load PrgEnv-gnu/8.5.0 \ - gcc/12.2.0 \ - cray-libsci/23.05.1.4 \ + gcc-native/12.3 \ + cray-libsci/23.12.5 \ craype-accel-host \ - cray-mpich/8.1.26 \ - craype \ + craype/2.7.30 \ + cray-mpich/8.1.28 \ cmake/3.27.7 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} setenv MPICH_ENV_DISPLAY 1 @@ -60,4 +62,4 @@ setenv MPICH_COLL_SYNC MPI_Bcast # for standalone MPAS builds setenv GNU_CRAY_LDFLAGS "-Wl,--enable-new-dtags" -setenv LD_LIBRARY_PATH="/opt/cray/pe/gcc/12.2.0/snos/lib64:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" +setenv LD_LIBRARY_PATH="/usr/lib64/gcc/x86_64-suse-linux/12:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" diff --git a/mache/spack/chicoma-cpu_gnu_mpich.sh b/mache/spack/chicoma-cpu_gnu_mpich.sh index 27d64a5d..d3ec206e 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.sh +++ b/mache/spack/chicoma-cpu_gnu_mpich.sh @@ -12,6 +12,8 @@ module rm cray-hdf5-parallel \ cray-parallel-netcdf \ cray-netcdf \ cray-hdf5 \ + gcc \ + gcc-native \ intel \ intel-oneapi \ nvidia \ @@ -33,16 +35,16 @@ module rm cray-hdf5-parallel \ # we must load cray-libsci for gcc to work module load PrgEnv-gnu/8.5.0 \ - gcc/12.2.0 \ - cray-libsci/23.05.1.4 \ + gcc-native/12.3 \ + cray-libsci/23.12.5 \ craype-accel-host \ - cray-mpich/8.1.26 \ - craype \ + craype/2.7.30 \ + cray-mpich/8.1.28 \ cmake/3.27.7 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} export MPICH_ENV_DISPLAY=1 @@ -57,7 +59,7 @@ export PNETCDF_HINTS="romio_ds_write=disable;romio_ds_read=disable;romio_cb_writ export FI_CXI_RX_MATCH_MODE=software export MPICH_COLL_SYNC=MPI_Bcast -export LD_LIBRARY_PATH="/opt/cray/pe/gcc/12.2.0/snos/lib64:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" +export LD_LIBRARY_PATH="/usr/lib64/gcc/x86_64-suse-linux/12:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" # for standalone MPAS builds export GNU_CRAY_LDFLAGS="-Wl,--enable-new-dtags" diff --git a/mache/spack/chicoma-cpu_gnu_mpich.yaml b/mache/spack/chicoma-cpu_gnu_mpich.yaml index b50ef62c..83c4f071 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.yaml +++ b/mache/spack/chicoma-cpu_gnu_mpich.yaml @@ -14,10 +14,10 @@ spack: unify: true packages: all: - compiler: [gcc@12.2.0] + compiler: [gcc@12.3] providers: - mpi: [cray-mpich@8.1.26] - lapack: [cray-libsci@23.05.1.4] + mpi: [cray-mpich@8.1.28] + lapack: [cray-libsci@23.12.5] bzip2: externals: - spec: bzip2@1.0.6 @@ -92,55 +92,55 @@ spack: buildable: false gcc: externals: - - spec: gcc@12.2.0 + - spec: gcc@12.3 modules: - PrgEnv-gnu/8.5.0 - - gcc/12.2.0 + - gcc-native/12.3 - libfabric/1.15.2.0 - - cray-libsci/23.05.1.4 - - craype + - cray-libsci/23.12.5 + - craype/2.7.30 - craype-accel-host - craype-x86-rome buildable: false cray-mpich: externals: - - spec: cray-mpich@8.1.26 + - spec: cray-mpich@8.1.28 modules: - - cray-mpich/8.1.26 + - cray-mpich/8.1.28 buildable: false cray-libsci: externals: - - spec: cray-libsci@23.05.1.4 + - spec: cray-libsci@23.12.5 modules: - - cray-libsci/23.05.1.4 + - cray-libsci/23.12.5 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/gnu/9.1 + - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/GNU/9.1/ + - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1 + - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 buildable: false netcdf-fortran: externals: - spec: netcdf-fortran@4.5.3 - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 buildable: false {% endif %} config: install_missing_compilers: false compilers: - compiler: - spec: gcc@12.2.0 + spec: gcc@12.3 paths: cc: cc cxx: CC @@ -151,10 +151,10 @@ spack: target: x86_64 modules: - PrgEnv-gnu/8.5.0 - - gcc/12.2.0 + - gcc-native/12.3 - libfabric/1.15.2.0 - - cray-libsci/23.05.1.4 - - craype + - cray-libsci/23.12.5 + - craype/2.7.30 - craype-accel-host - craype-x86-rome environment: From 6abd81bf7b4fda2e186ed7f16e938c630e1d0de9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 21 Oct 2024 15:05:28 -0600 Subject: [PATCH 13/13] Update to newer nvidia modules --- mache/spack/chicoma-cpu_nvidia_mpich.csh | 14 ++++++----- mache/spack/chicoma-cpu_nvidia_mpich.sh | 14 ++++++----- mache/spack/chicoma-cpu_nvidia_mpich.yaml | 29 +++++++++++------------ 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.csh b/mache/spack/chicoma-cpu_nvidia_mpich.csh index c19c29d2..3283bba4 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.csh +++ b/mache/spack/chicoma-cpu_nvidia_mpich.csh @@ -13,6 +13,8 @@ module rm cray-hdf5-parallel \ cray-parallel-netcdf \ cray-netcdf \ cray-hdf5 \ + gcc \ + gcc-native \ intel \ intel-oneapi \ nvidia \ @@ -34,15 +36,15 @@ module rm cray-hdf5-parallel \ module load PrgEnv-nvidia/8.5.0 \ nvidia/24.7 \ - cray-libsci/23.05.1.4 \ + cray-libsci/23.12.5 \ craype-accel-host \ - cray-mpich/8.1.26 \ - craype \ + craype/2.7.30 \ + cray-mpich/8.1.28 \ cmake/3.27.7 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} setenv MPICH_ENV_DISPLAY 1 diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.sh b/mache/spack/chicoma-cpu_nvidia_mpich.sh index bd2582c7..528dac5f 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.sh +++ b/mache/spack/chicoma-cpu_nvidia_mpich.sh @@ -12,6 +12,8 @@ module rm cray-hdf5-parallel \ cray-parallel-netcdf \ cray-netcdf \ cray-hdf5 \ + gcc \ + gcc-native \ intel \ intel-oneapi \ nvidia \ @@ -33,15 +35,15 @@ module rm cray-hdf5-parallel \ module load PrgEnv-nvidia/8.5.0 \ nvidia/24.7 \ - cray-libsci/23.05.1.4 \ + cray-libsci/23.12.5 \ craype-accel-host \ - cray-mpich/8.1.26 \ - craype \ + craype/2.7.30 \ + cray-mpich/8.1.28 \ cmake/3.27.7 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} export MPICH_ENV_DISPLAY=1 diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.yaml b/mache/spack/chicoma-cpu_nvidia_mpich.yaml index 7cd2d934..07590af2 100644 --- a/mache/spack/chicoma-cpu_nvidia_mpich.yaml +++ b/mache/spack/chicoma-cpu_nvidia_mpich.yaml @@ -15,8 +15,8 @@ spack: all: compiler: [nvhpc@24.7] providers: - mpi: [cray-mpich@8.1.26] - lapack: [cray-libsci@23.05.1.4] + mpi: [cray-mpich@8.1.28] + lapack: [cray-libsci@23.12.5] bzip2: externals: - spec: bzip2@1.0.6 @@ -91,37 +91,36 @@ spack: buildable: false cray-mpich: externals: - - spec: cray-mpich@8.1.26 + - spec: cray-mpich@8.1.28 modules: - - cray-mpich/8.1.26 + - cray-mpich/8.1.28 buildable: false cray-libsci: externals: - - spec: cray-libsci@23.05.1.4 + - spec: cray-libsci@23.12.5 modules: - - cray-libsci/23.05.1.4 + - cray-libsci/23.12.5 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared - modules: - - cray-hdf5-parallel/1.12.2.3 + - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/nvidia/23.3 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/nvidia/20.7 + - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/nvidia/23.3 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7 + - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3 buildable: false netcdf-fortran: externals: - spec: netcdf-fortran@4.5.3 - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3 buildable: false {% endif %} config: @@ -141,7 +140,7 @@ spack: - PrgEnv-nvidia/8.5.0 - nvidia/24.7 - libfabric/1.15.2.0 - - cray-libsci/23.05.1.4 + - cray-libsci/23.12.5 - craype - craype-accel-host - craype-x86-rome