From 9309c4a5cd950d7c731c2c433dc94ee508a6d02b Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 2 Dec 2023 19:21:01 -0700 Subject: [PATCH 1/2] Use only polaris release version is spack env name This means that we update the release version for new spack envs and we update the alpha, beta, etc. for dependency changes. --- configure_polaris_envs.py | 2 +- deploy/bootstrap.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/configure_polaris_envs.py b/configure_polaris_envs.py index c4125d1c7..43318b924 100755 --- a/configure_polaris_envs.py +++ b/configure_polaris_envs.py @@ -43,7 +43,7 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate, channels = '--use-local' else: channels = '' - packages = f'progressbar2 jinja2 {mache}' + packages = f'jinja2 {mache} packaging progressbar2' if recreate or not os.path.exists(env_path): print('Setting up a conda environment for installing polaris\n') commands = f'{activate_base} && ' \ diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py index 24b70355a..a223bb1ed 100755 --- a/deploy/bootstrap.py +++ b/deploy/bootstrap.py @@ -17,6 +17,7 @@ from mache import MachineInfo from mache import discover_machine as mache_discover_machine from mache.spack import get_spack_script, make_spack_env +from packaging import version from shared import ( check_call, get_conda_base, @@ -210,7 +211,9 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, check_supported('petsc', machine, compiler, mpi, source_path) if env_type == 'dev': - spack_env = f'dev_polaris_{polaris_version}{env_suffix}' + ver = version.parse(polaris_version) + release_version = '.'.join(str(vr) for vr in ver.release) + spack_env = f'dev_polaris_{release_version}{env_suffix}' elif env_type == 'test_release': spack_env = f'test_polaris_{polaris_version}{env_suffix}' else: From cf29ebc5b8ca2064ffc2473c591fe37f61aebac8 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 17 Jan 2024 02:05:22 -0800 Subject: [PATCH 2/2] Update docs on deploying spack --- docs/developers_guide/deploying_spack.md | 28 +++++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/developers_guide/deploying_spack.md b/docs/developers_guide/deploying_spack.md index 7d11d3652..55ffdbf59 100644 --- a/docs/developers_guide/deploying_spack.md +++ b/docs/developers_guide/deploying_spack.md @@ -15,6 +15,10 @@ If system modules change in E3SM, we try to stay in sync: - pnetcdf - mkl (or other linear algebra libs) +When we update the `mache` version in Polaris, we also need to bump the +Polaris version (typically either the major or the minor version) and then +re-deploy shared spack environments on each supported machine. + ### Spack Spack is for libraries used by MPAS and tools that need system MPI: @@ -25,6 +29,10 @@ Spack is for libraries used by MPAS and tools that need system MPI: - PETSc - Netlib LAPACK +When we update the versions of any of these libraries in Polaris, we also need +to bump the Polaris version (typically either the major or the minor version) +and then re-deploy shared spack environments on each supported machine. + ### Conda Conda (via conda-forge) is used for python packages and related dependencies @@ -32,6 +40,10 @@ that don’t need system MPI. Conda environments aren’t shared between developers because the polaris you’re developing is part of the conda environment. +When we update the constraints on conda dependencies, we also need to bump the +Polaris alpha, beta or rc version. We do not need to re-deploy spack +environments on share machines because they remain unaffected. + ## Mache A brief tour of mache. @@ -100,14 +112,18 @@ by polaris. Here, we will use `` as a stand-in for the fork of mache to use (e.g. `E3SM-Project/mache`) and `` as the stand-in for a branch on that fork (e.g. `main`). -We also need to make sure there is a spack branch for the version of mache. -This is a branch off of the develop branch on +We also need to make sure there is a spack branch for the version of Polaris. +The spack branch is a branch off of the develop branch on [E3SM’s spack repo](https://github.com/E3SM-Project/spack) that has any -updates to packages required for this version of mache and polaris. The -spack branch will omit any alpha, beta or rc suffix on the mache version -because it is intended to be the spack branch we will use once the `mache` -release happens. In this example, we will work with the branch +uupdates to packages required for this version of mache. The remote branch +is named after the release version of mache (omitting any alpha, beta or rc +suffix because it is intended to be the spack branch we will use once the +``mache`` release happens). In this example, we will work with the branch [spack_for_mache_1.12.0](https://github.com/E3SM-Project/spack/tree/spack_for_mache_1.12.0). +The local clone is instead named after the Polaris version (again any omitting +alpha, beta or rc) plus the compiler and MPI library because we have discovered +two users cannot make modifications to the same git clone. Giving each clone +of the spack branch a unique name ensures that they are independent. Here's how to get a branch of polaris we're testing (`simplify_local_mache` in this case) as a local worktree: