Merge pull request #201 from altheaden/update-chicoma-cpu

Update `chicoma-cpu` modules
E3SM-Project · Oct 23, 2024 · 4f0b8ee · 4f0b8ee
2 parents 353766f + 6abd81b
commit 4f0b8ee
Show file tree

Hide file tree

Showing 6 changed files with 216 additions and 176 deletions.
diff --git a/mache/spack/chicoma-cpu_gnu_mpich.csh b/mache/spack/chicoma-cpu_gnu_mpich.csh
@@ -5,31 +5,46 @@ setenv HTTP_PROXY http://proxyout.lanl.gov:8080
 setenv HTTPS_PROXY http://proxyout.lanl.gov:8080
 setenv FTP_PROXY http://proxyout.lanl.gov:8080
 
-source /usr/share/lmod/8.3.1/init/csh
+source /usr/share/lmod/lmod/init/csh
 
-module rm PrgEnv-gnu
-module rm PrgEnv-nvidia
-module rm PrgEnv-cray
-module rm PrgEnv-aocc
-module rm craype-accel-nvidia80
-module rm craype-accel-host
+module rm cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          cray-netcdf \
+          cray-hdf5 \
+          gcc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          nvidia \
+          aocc \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          craype \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc
 
-module load PrgEnv-gnu/8.4.0
-module load gcc/12.2.0
-module load craype-accel-host
-{% if e3sm_lapack %}
-module load cray-libsci
-{% endif %}
-module load craype
-module load libfabric/1.15.2.0
-module load cray-mpich/8.1.26
+# we must load cray-libsci for gcc to work
+module load PrgEnv-gnu/8.5.0 \
+            gcc-native/12.3 \
+            cray-libsci/23.12.5 \
+            craype-accel-host \
+            craype/2.7.30 \
+            cray-mpich/8.1.28 \
+            cmake/3.27.7
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel
-module rm cray-netcdf-hdf5parallel
-module rm cray-parallel-netcdf
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 setenv MPICH_ENV_DISPLAY 1
@@ -44,4 +59,7 @@ setenv PNETCDF_HINTS "romio_ds_write=disable;romio_ds_read=disable;romio_cb_writ
 setenv FI_CXI_RX_MATCH_MODE software
 setenv MPICH_COLL_SYNC MPI_Bcast
 
-setenv LD_LIBRARY_PATH $CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
+# for standalone MPAS builds
+setenv GNU_CRAY_LDFLAGS "-Wl,--enable-new-dtags"
+
+setenv LD_LIBRARY_PATH="/usr/lib64/gcc/x86_64-suse-linux/12:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}"
diff --git a/mache/spack/chicoma-cpu_gnu_mpich.sh b/mache/spack/chicoma-cpu_gnu_mpich.sh
@@ -5,31 +5,46 @@ export HTTP_PROXY=http://proxyout.lanl.gov:8080
 export HTTPS_PROXY=http://proxyout.lanl.gov:8080
 export FTP_PROXY=http://proxyout.lanl.gov:8080
 
-source /usr/share/lmod/8.3.1/init/sh
+source /usr/share/lmod/lmod/init/sh
 
-module rm PrgEnv-gnu
-module rm PrgEnv-nvidia
-module rm PrgEnv-cray
-module rm PrgEnv-aocc
-module rm craype-accel-nvidia80
-module rm craype-accel-host
+module rm cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          cray-netcdf \
+          cray-hdf5 \
+          gcc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          nvidia \
+          aocc \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          craype \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc
 
-module load PrgEnv-gnu/8.4.0
-module load gcc/12.2.0
-module load craype-accel-host
-{% if e3sm_lapack %}
-module load cray-libsci
-{% endif %}
-module load craype
-module load libfabric/1.15.2.0
-module load cray-mpich/8.1.26
+# we must load cray-libsci for gcc to work
+module load PrgEnv-gnu/8.5.0 \
+            gcc-native/12.3 \
+            cray-libsci/23.12.5 \
+            craype-accel-host \
+            craype/2.7.30 \
+            cray-mpich/8.1.28 \
+            cmake/3.27.7
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel
-module rm cray-netcdf-hdf5parallel
-module rm cray-parallel-netcdf
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 export MPICH_ENV_DISPLAY=1
@@ -44,4 +59,7 @@ export PNETCDF_HINTS="romio_ds_write=disable;romio_ds_read=disable;romio_cb_writ
 export FI_CXI_RX_MATCH_MODE=software
 export MPICH_COLL_SYNC=MPI_Bcast
 
-export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH="/usr/lib64/gcc/x86_64-suse-linux/12:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}"
+
+# for standalone MPAS builds
+export GNU_CRAY_LDFLAGS="-Wl,--enable-new-dtags"
diff --git a/mache/spack/chicoma-cpu_gnu_mpich.yaml b/mache/spack/chicoma-cpu_gnu_mpich.yaml
@@ -2,9 +2,7 @@ spack:
   specs:
   - gcc
   - cray-mpich
-{% if e3sm_lapack %}
   - cray-libsci
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
   - hdf5
   - netcdf-c
@@ -16,17 +14,20 @@ spack:
     unify: true
   packages:
     all:
-      compiler: [gcc@12.2.0]
+      compiler: [gcc@12.3]
       providers:
-        mpi: [[email protected]]
-{% if e3sm_lapack %}
-        lapack: [[email protected]]
-{% endif %}
+        mpi: [[email protected]]
+        lapack: [[email protected]]
     bzip2:
       externals:
       - spec: [email protected]
         prefix: /usr
       buildable: false
+    cmake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr/projects/hpcsoft/tce/23-05/cos2-x86_64-cc80/packages/cmake/cmake-3.27.7/
+      buildable: false
     curl:
       externals:
       - spec: [email protected]
@@ -91,67 +92,55 @@ spack:
       buildable: false
     gcc:
       externals:
-      - spec: gcc@12.2.0
+      - spec: gcc@12.3
         modules:
-        - PrgEnv-gnu/8.4.0
-        - gcc/12.2.0
-        - craype-accel-host
-        - craype
+        - PrgEnv-gnu/8.5.0
+        - gcc-native/12.3
         - libfabric/1.15.2.0
+        - cray-libsci/23.12.5
+        - craype/2.7.30
+        - craype-accel-host
+        - craype-x86-rome
       buildable: false
     cray-mpich:
       externals:
-      - spec: [email protected]
-        prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/9.1
+      - spec: [email protected]
         modules:
-        - libfabric/1.15.2.0
-        - cray-mpich/8.1.26
+        - cray-mpich/8.1.28
       buildable: false
-    libfabric:
-      externals:
-      - spec: [email protected]
-        prefix: /opt/cray/libfabric/1.15.2.0
-        modules:
-        - libfabric/1.15.2.0
-      buildable: false
-{% if e3sm_lapack %}
     cray-libsci:
       externals:
-      - spec: [email protected]
-        prefix: /opt/cray/pe/libsci/23.05.1.4/GNU/9.1/x86_64
+      - spec: [email protected]
         modules:
-        - cray-libsci/23.05.1.4
+        - cray-libsci/23.12.5
       buildable: false
-{% endif %}
 {% if e3sm_hdf5_netcdf %}
     hdf5:
       externals:
-      - spec: [email protected]~cxx+fortran+hl~java+mpi+shared
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1
-        modules:
-        - cray-hdf5-parallel/1.12.2.3
+      - spec: [email protected]~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3
       buildable: false
     parallel-netcdf:
       externals:
-      - spec: [email protected].3+cxx+fortran+pic+shared
-        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/GNU/9.1/
+      - spec: [email protected].9+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3
       buildable: false
     netcdf-c:
       externals:
-      - spec: [email protected].3+mpi~parallel-netcdf
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
+      - spec: [email protected].9+mpi~parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
       buildable: false
     netcdf-fortran:
       externals:
       - spec: [email protected]
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
       buildable: false
 {% endif %}
   config:
     install_missing_compilers: false
   compilers:
   - compiler:
-      spec: gcc@12.2.0
+      spec: gcc@12.3
       paths:
         cc: cc
         cxx: CC
@@ -161,11 +150,13 @@ spack:
       operating_system: sles15
       target: x86_64
       modules:
-      - PrgEnv-gnu/8.4.0
-      - gcc/12.2.0
-      - craype-accel-host
-      - craype
+      - PrgEnv-gnu/8.5.0
+      - gcc-native/12.3
       - libfabric/1.15.2.0
+      - cray-libsci/23.12.5
+      - craype/2.7.30
+      - craype-accel-host
+      - craype-x86-rome
       environment:
         prepend_path:
           PKG_CONFIG_PATH: "/opt/cray/xpmem/2.5.2-2.4_3.45__gd0f7936.shasta/lib64/pkgconfig"
diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.csh b/mache/spack/chicoma-cpu_nvidia_mpich.csh
@@ -5,32 +5,46 @@ setenv HTTP_PROXY http://proxyout.lanl.gov:8080
 setenv HTTPS_PROXY http://proxyout.lanl.gov:8080
 setenv FTP_PROXY http://proxyout.lanl.gov:8080
 
-source /usr/share/lmod/8.3.1/init/csh
+source /usr/share/lmod/lmod/init/csh
 
-module rm PrgEnv-gnu
-module rm PrgEnv-nvidia
-module rm PrgEnv-cray
-module rm PrgEnv-aocc
-module rm craype-accel-nvidia80
-module rm craype-accel-host
 
-module load PrgEnv-nvidia/8.4.0
-module load nvidia/22.7
-module load craype-x86-milan
-module load libfabric/1.15.2.0
-module load craype-accel-host
-module load craype
-module load cray-mpich/8.1.26
-{% if e3sm_lapack %}
-module load cray-libsci/23.05.1.4
-{% endif %}
+module rm cray-hdf5-parallel \
+          cray-netcdf-hdf5parallel \
+          cray-parallel-netcdf \
+          cray-netcdf \
+          cray-hdf5 \
+          gcc \
+          gcc-native \
+          intel \
+          intel-oneapi \
+          nvidia \
+          aocc \
+          cudatoolkit \
+          climate-utils \
+          cray-libsci \
+          craype \
+          craype-accel-nvidia80 \
+          craype-accel-host \
+          perftools-base \
+          perftools \
+          darshan \
+          PrgEnv-gnu \
+          PrgEnv-intel \
+          PrgEnv-nvidia \
+          PrgEnv-cray \
+          PrgEnv-aocc
+
+module load PrgEnv-nvidia/8.5.0 \
+            nvidia/24.7 \
+            cray-libsci/23.12.5 \
+            craype-accel-host \
+            craype/2.7.30 \
+            cray-mpich/8.1.28 \
+            cmake/3.27.7
 {% if e3sm_hdf5_netcdf %}
-module rm cray-hdf5-parallel
-module rm cray-netcdf-hdf5parallel
-module rm cray-parallel-netcdf
-module load cray-hdf5-parallel/1.12.2.3
-module load cray-netcdf-hdf5parallel/4.9.0.3
-module load cray-parallel-netcdf/1.12.3.3
+module load cray-hdf5-parallel/1.12.2.9 \
+            cray-netcdf-hdf5parallel/4.9.0.9 \
+            cray-parallel-netcdf/1.12.3.9
 {% endif %}
 
 setenv MPICH_ENV_DISPLAY 1