From 688c643c6793f950e9b765b82e45c51a2287c6f7 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 2 Feb 2023 10:41:20 -0600 Subject: [PATCH 1/4] Update cime machine config file This is from E3SM master hash 9bb3b4f3a09ac59fbc93b8ab317ca0fa6a6f18bb --- mache/cime_machine_config/config_machines.xml | 169 +++++++++++++----- 1 file changed, 124 insertions(+), 45 deletions(-) diff --git a/mache/cime_machine_config/config_machines.xml b/mache/cime_machine_config/config_machines.xml index 07205e09..132a843c 100644 --- a/mache/cime_machine_config/config_machines.xml +++ b/mache/cime_machine_config/config_machines.xml @@ -104,34 +104,25 @@ cray-hdf5 cray-netcdf cray-netcdf-hdf5parallel - craype/2.7.12 - - - PrgEnv-gnu - PrgEnv-cray - PrgEnv-intel/8.1.0 - intel/19.1.0.166 + craype/2.7.15 PrgEnv-cray - PrgEnv-gnu/8.1.0 - gcc/9.3.0 + PrgEnv-gnu + PrgEnv-gnu/8.3.3 + gcc/12.1.0 - PrgEnv-intel PrgEnv-gnu gcc/9.3.0 - PrgEnv-cray/8.1.0 + PrgEnv-cray/8.3.3 darshan - - cray-libsci/21.06.1.1 - - cray-mpich/8.1.11 - cray-hdf5-parallel/1.12.0.7 - cray-netcdf-hdf5parallel/4.7.4.7 - cray-parallel-netcdf/1.12.1.7 + cray-mpich/8.1.16 + cray-hdf5-parallel/1.12.1.3 + cray-netcdf-hdf5parallel/4.8.1.3 + cray-parallel-netcdf/1.12.2.3 @@ -140,20 +131,14 @@ 0.1 1000 - /usr/lib/perl5/5.26.1 - opt/cray/pe/netcdf-hdf5parallel/4.7.4.4/gnu/9.1/ - opt/cray/pe/netcdf-hdf5parallel/4.7.4.4/gnu/9.1/ + /usr/lib/perl5/5.26.2 + /opt/cray/pe/netcdf-hdf5parallel/4.8.1.3/gnu/9.1/ + /opt/cray/pe/netcdf-hdf5parallel/4.8.1.3/gnu/9.1/ $SHELL{dirname $(dirname $(which pnetcdf_version))} 128M - - granularity=core,balanced - - - granularity=thread,balanced - cores @@ -596,19 +581,16 @@ 1 slurm e3sm - 64 - 64 + 56 + 56 TRUE srun -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - --threads-per-core=1 -c $ENV{OMP_NUM_THREADS} -m *:block - @@ -628,7 +610,7 @@ PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.1.0 + amd amd/5.4.0 @@ -1945,7 +1927,7 @@ srun --mpi=pmi2 -l -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit - $SHELL{if [ 64 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} + $SHELL{if [ 64 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -c $SHELL{echo 128/ {{ tasks_per_node }} |bc} -m plane={{ tasks_per_node }} @@ -4214,9 +4196,106 @@ - - Google Cloud cluster with c2-compute-60's - gcp* + + Google Cloud cluster using compute nodes c2d-compute-112's gcpe3sm12 + gcpe3sm12* + LINUX + gnu + openmpi + /home/$USER/e3sm/scratch + /home/inputdata + /home/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /home/baselines/$COMPILER + /home/tools/cprnc/cprnc + 24 + e3sm_developer + 8 + slurm + e3sm + 112 + 56 + FALSE + + srun + + --mpi=pmi2 + --label + -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit + -c $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/ {{ tasks_per_node }} |bc} + $SHELL{if [ `./xmlquery --value MAX_TASKS_PER_NODE` -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} + -m plane={{ tasks_per_node }} + + + + + /usr/share/lmod/lmod/init/env_modules_python.py + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh + + /usr/share/lmod/lmod/libexec/lmod python + module + module + + + /opt/apps/spack/share/spack/modules/linux-centos7-zen2 + /opt/apps/spack/share/spack/modules/linux-centos7-x86_64_v3 + gcc + openmpi + binutils + netlib-lapack + openblas + hdf5 + netcdf-c + parallel-netcdf + + + + gcc/12.2.0 + + + + openmpi/4.1.4 + + + + cmake + perl + perl-xml-libxml + netcdf-c + netcdf-cxx + netcdf-fortran + parallel-netcdf + hdf5 + netlib-lapack + openblas + + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.2 + 0.20 + + $SHELL{dirname $(dirname $(which h5diff))} + $SHELL{dirname $(dirname $(which nc-config))} + $SHELL{dirname $(dirname $(which nf-config))} + $SHELL{dirname $(dirname $(which pnetcdf-config))} + /opt/apps/spack/opt/spack/linux-centos7-zen2/gcc-12.2.0/openblas-0.3.21-z66r7lyxwkhsshgreexm4cedffp73scp + /opt/apps/spack/opt/spack/linux-centos7-zen2/gcc-12.2.0/netlib-lapack-3.10.1-lkhddpuidlw2z74g5ui6eq5iattsfjxp + $ENV{PERL5LIB}:/opt/apps/spack/opt/spack/linux-centos7-zen2/gcc-12.2.0/perl-5.36.0-sly2pft2edg2p3iyijfyy6dzntusokno/lib/site_perl/5.36.0 + FALSE + + + + 128M + threads + + + + + Google Cloud cluster with c2-compute-60's gcp-e3sm10 + gcp-e3sm10* LINUX gnu openmpi @@ -4239,7 +4318,7 @@ --mpi=pmi2 --label - -n {{ total_tasks }} -N {{ num_nodes }} + -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit -c $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/ {{ tasks_per_node }} |bc} $SHELL{if [ `./xmlquery --value MAX_TASKS_PER_NODE` -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane={{ tasks_per_node }} @@ -4273,13 +4352,13 @@ cmake perl perl-xml-libxml - netcdf-c-gcc@12.2.0 - netcdf-cxx-gcc@12.2.0 - netcdf-fortran-gcc@12.2.0 - parallel-netcdf-gcc@12.2.0 - hdf5-gcc@12.2.0 - netlib-lapack-gcc@12.2.0 - openblas-gcc@12.2.0 + netcdf-c-gcc@12.2.0 + netcdf-cxx-gcc@12.2.0 + netcdf-fortran-gcc@12.2.0 + parallel-netcdf-gcc@12.2.0 + hdf5-gcc@12.2.0 + netlib-lapack-gcc@12.2.0 + openblas-gcc@12.2.0 From cab02e8cd6447b1cd38f2779fa0d159c499b00ba Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 2 Feb 2023 09:49:42 -0700 Subject: [PATCH 2/4] Update to v1.12.0rc1 --- conda/meta.yaml | 4 ++-- mache/version.py | 4 ++-- setup.cfg | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/meta.yaml b/conda/meta.yaml index 0fa89312..b2ff03ec 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "mache" %} -{% set version = "1.11.0" %} +{% set version = "1.12.0rc1" %} package: name: {{ name|lower }} @@ -45,4 +45,4 @@ about: extra: recipe-maintainers: - - xylar + - xylar \ No newline at end of file diff --git a/mache/version.py b/mache/version.py index 1e050674..ba739136 100644 --- a/mache/version.py +++ b/mache/version.py @@ -1,2 +1,2 @@ -__version_info__ = (1, 11, 0) -__version__ = '.'.join(str(vi) for vi in __version_info__) +__version_info__ = (1, 12, 0) +__version__ = '.'.join(str(vi) for vi in __version_info__) \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 406abf36..d1ebd4d5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = mache -version = 1.11.0 +version = 1.12.0rc1 author = Xylar Asay-Davis author_email = xylar@lanl.gov description = A package for providing configuration data relate to E3SM supported machines @@ -27,4 +27,4 @@ install_requires = [options.entry_points] console_scripts = - mache = mache.__main__:main + mache = mache.__main__:main \ No newline at end of file From 8fa0b2895eb625cba43ae5a1c73f667ea1fa63d3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 5 Feb 2023 11:38:37 -0700 Subject: [PATCH 3/4] Suppress confusing error message When we try to remove a spack environment but it doesn't exist, there was a confusing error message that this merge aims to suppress. --- mache/spack/build_spack_env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mache/spack/build_spack_env.template b/mache/spack/build_spack_env.template index efbabc5b..ca30b4a1 100644 --- a/mache/spack/build_spack_env.template +++ b/mache/spack/build_spack_env.template @@ -19,7 +19,7 @@ source share/spack/setup-env.sh spack mirror add spack_mirror file://{{ spack_mirror }} {% endif %} -spack env remove -y {{ env_name }} && \ +spack env remove -y {{ env_name }} >& /dev/null && \ echo "recreating environment: {{ env_name }}" || \ echo "creating new environment: {{ env_name }}" spack env create {{ env_name }} {{ yaml_filename }} From 3b68b5559f36566c008548d7cc37c9b73ed7b60a Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 14 Feb 2023 21:37:14 -0700 Subject: [PATCH 4/4] Change chrysalis os rhel8 --> centos8 --- mache/spack/chrysalis_gnu_openmpi.yaml | 2 +- mache/spack/chrysalis_intel_impi.yaml | 2 +- mache/spack/chrysalis_intel_openmpi.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mache/spack/chrysalis_gnu_openmpi.yaml b/mache/spack/chrysalis_gnu_openmpi.yaml index c6de3a5f..e6040bd4 100644 --- a/mache/spack/chrysalis_gnu_openmpi.yaml +++ b/mache/spack/chrysalis_gnu_openmpi.yaml @@ -139,7 +139,7 @@ spack: f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-9.2.0-ugetvbp/bin/gfortran fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-9.2.0-ugetvbp/bin/gfortran flags: {} - operating_system: rhel8 + operating_system: centos8 target: x86_64 modules: [] environment: {} diff --git a/mache/spack/chrysalis_intel_impi.yaml b/mache/spack/chrysalis_intel_impi.yaml index cd79fb08..2e562294 100644 --- a/mache/spack/chrysalis_intel_impi.yaml +++ b/mache/spack/chrysalis_intel_impi.yaml @@ -139,7 +139,7 @@ spack: f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort flags: {} - operating_system: rhel8 + operating_system: centos8 target: x86_64 modules: [] environment: {} diff --git a/mache/spack/chrysalis_intel_openmpi.yaml b/mache/spack/chrysalis_intel_openmpi.yaml index 0c135552..f81973e7 100644 --- a/mache/spack/chrysalis_intel_openmpi.yaml +++ b/mache/spack/chrysalis_intel_openmpi.yaml @@ -139,7 +139,7 @@ spack: f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort flags: {} - operating_system: rhel8 + operating_system: centos8 target: x86_64 modules: [] environment: {}