diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py index 852519fb1..9121147e3 100755 --- a/deploy/bootstrap.py +++ b/deploy/bootstrap.py @@ -511,6 +511,11 @@ def build_spack_soft_env(config, update_spack, machine, env_type, # noqa: C901 esmf = config.get('deploy', 'esmf') + if config.has_option('deploy', 'spack_mirror'): + spack_mirror = config.get('deploy', 'spack_mirror') + else: + spack_mirror = None + spack_branch_base = f'{spack_base}/{spack_env}' specs = list() @@ -545,7 +550,8 @@ def build_spack_soft_env(config, update_spack, machine, env_type, # noqa: C901 spack_specs=specs, compiler=compiler, mpi=mpi, machine=machine, config_file=machine_config, include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, - yaml_template=yaml_template, tmpdir=tmpdir) + yaml_template=yaml_template, tmpdir=tmpdir, + spack_mirror=spack_mirror) spack_view = f'{spack_branch_base}/var/spack/environments/' \ f'{spack_env}/.spack-env/view' @@ -593,12 +599,12 @@ def build_spack_libs_env(config, update_spack, machine, compiler, # noqa: C901 else: include_e3sm_lapack = True if metis != 'None': - specs.append(f'"metis@{metis}~shared"') + specs.append(f'"metis@{metis}+int64+real64~shared"') if moab != 'None': specs.append( f'"moab@{moab}+mpi+hdf5+netcdf+pnetcdf+metis+parmetis+tempest"') if parmetis != 'None': - specs.append(f'"parmetis@{parmetis}~shared"') + specs.append(f'"parmetis@{parmetis}+int64~shared"') if petsc != 'None': specs.append(f'"petsc@{petsc}+mpi+batch"') diff --git a/deploy/default.cfg b/deploy/default.cfg index 4c0c922b2..2a3caca2f 100644 --- a/deploy/default.cfg +++ b/deploy/default.cfg @@ -21,7 +21,7 @@ mpi = nompi # versions of conda packages geometric_features = 1.2.0 -mache = 1.19.0 +mache = 1.24.0 mpas_tools = 0.34.1 otps = 2021.10 parallelio = 2.6.0 diff --git a/e3sm_submodules/E3SM-Project b/e3sm_submodules/E3SM-Project index 1fb279e59..04842fecc 160000 --- a/e3sm_submodules/E3SM-Project +++ b/e3sm_submodules/E3SM-Project @@ -1 +1 @@ -Subproject commit 1fb279e59d0d4d9fcae708fdba80615d0c516319 +Subproject commit 04842fecc6341266ec9bf5a89dda99b9903d5b6f diff --git a/e3sm_submodules/MALI-Dev b/e3sm_submodules/MALI-Dev index ace160c64..e23c40196 160000 --- a/e3sm_submodules/MALI-Dev +++ b/e3sm_submodules/MALI-Dev @@ -1 +1 @@ -Subproject commit ace160c64167a3a40d7fc9afee6c6b2efd8b6a50 +Subproject commit e23c401962f26fa2a9aee62eff3f54004aa759b0 diff --git a/e3sm_submodules/Omega b/e3sm_submodules/Omega index e55160261..e9b9f4def 160000 --- a/e3sm_submodules/Omega +++ b/e3sm_submodules/Omega @@ -1 +1 @@ -Subproject commit e551602611e4afb24b208f285eee0f4c7b5bc117 +Subproject commit e9b9f4def69870b60d841fc65a298e73753f86fa diff --git a/polaris/machines/chicoma-cpu.cfg b/polaris/machines/chicoma-cpu.cfg index 26a71e909..8fe0455d8 100644 --- a/polaris/machines/chicoma-cpu.cfg +++ b/polaris/machines/chicoma-cpu.cfg @@ -28,6 +28,9 @@ spack = /usr/projects/e3sm/polaris/chicoma-cpu/spack # pnetcdf as E3SM (spack modules are used otherwise) use_e3sm_hdf5_netcdf = True +# location of a spack mirror for polaris to use +spack_mirror = /usr/projects/e3sm/polaris/chicoma-cpu/spack/spack_mirror + # The parallel section describes options related to running jobs in parallel [parallel] @@ -42,6 +45,11 @@ cores_per_node = 128 # hanging on perlmutter) threads_per_core = 1 +# quality of service +# overriding mache because the debug qos also requires --reservaiton debug, +# which polaris doesn't currently support +qos = standard + # Config options related to creating a job script [job] diff --git a/polaris/version.py b/polaris/version.py index 9552fa7bf..45707517d 100644 --- a/polaris/version.py +++ b/polaris/version.py @@ -1 +1 @@ -__version__ = '0.3.0-alpha.2' +__version__ = '0.4.0-alpha.1' diff --git a/utils/omega/ctest/omega_ctest.py b/utils/omega/ctest/omega_ctest.py index f94a6b0b8..7abe4018f 100755 --- a/utils/omega/ctest/omega_ctest.py +++ b/utils/omega/ctest/omega_ctest.py @@ -13,7 +13,7 @@ def make_build_script(machine, compiler, branch, build_only, mesh_filename, planar_mesh_filename, sphere_mesh_filename, - debug, clean, cmake_flags): + debug, clean, cmake_flags, account): """ Make a shell script for checking out Omega and its submodules, building Omega and its ctests, linking to testing data files, and running ctests. @@ -45,6 +45,9 @@ def make_build_script(machine, compiler, branch, build_only, mesh_filename, if cmake_flags is None: cmake_flags = '' + if account is not None: + cmake_flags = f'{cmake_flags} -DOMEGA_CIME_PROJECT={account}' + script = template.render(update_omega_submodule=update_omega_submodule, polaris_source_dir=polaris_source_dir, omega_base_dir=branch, @@ -108,15 +111,16 @@ def download_meshes(config): return download_targets -def write_job_script(config, machine, compiler, submit): +def write_job_script(config, machine, compiler, account, submit): """ Write a job script for running the build script """ - if config.has_option('parallel', 'account'): - account = config.get('parallel', 'account') - else: - account = '' + if account is None: + if config.has_option('parallel', 'account'): + account = config.get('parallel', 'account') + else: + account = '' nodes = 1 @@ -190,6 +194,8 @@ def main(): help='Whether to only build Omega in debug mode') parser.add_argument('--cmake_flags', dest='cmake_flags', help='Quoted string with additional cmake flags') + parser.add_argument('--account', dest='account', + help='slurm account to submit the job to') args = parser.parse_args() @@ -206,6 +212,7 @@ def main(): debug = args.debug clean = args.clean cmake_flags = args.cmake_flags + account = args.account if 'SLURM_JOB_ID' in os.environ: # already on a comptue node so we will just run ctests directly @@ -221,7 +228,8 @@ def main(): build_only=build_only, mesh_filename=mesh_filename, planar_mesh_filename=planar_mesh_filename, sphere_mesh_filename=sphere_mesh_filename, - debug=debug, clean=clean, cmake_flags=cmake_flags) + debug=debug, clean=clean, cmake_flags=cmake_flags, + account=account) # clear environment variables and start fresh with those from login # so spack doesn't get confused by conda @@ -229,7 +237,7 @@ def main(): shell=True) write_job_script(config=config, machine=machine, compiler=compiler, - submit=submit) + account=account, submit=submit) if __name__ == '__main__':