From 90fad6771769edad0087eba27a91bef1c97c3682 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 10 Jul 2024 02:43:40 -0500 Subject: [PATCH 1/9] Update to 0.4.0-alpha.1 --- polaris/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polaris/version.py b/polaris/version.py index 9552fa7bf..45707517d 100644 --- a/polaris/version.py +++ b/polaris/version.py @@ -1 +1 @@ -__version__ = '0.3.0-alpha.2' +__version__ = '0.4.0-alpha.1' From 5a9d1f11b1ac60f3ef628a2e78923b1c6096f0c7 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 5 Apr 2024 04:17:05 -0500 Subject: [PATCH 2/9] Set METIS to build with 64-bit int and real This is necessary for larger meshes and partition sizes (such at the RRS meshe used in E3SM HR simulations). --- deploy/bootstrap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py index 852519fb1..e49963ba4 100755 --- a/deploy/bootstrap.py +++ b/deploy/bootstrap.py @@ -593,12 +593,12 @@ def build_spack_libs_env(config, update_spack, machine, compiler, # noqa: C901 else: include_e3sm_lapack = True if metis != 'None': - specs.append(f'"metis@{metis}~shared"') + specs.append(f'"metis@{metis}+int64+real64~shared"') if moab != 'None': specs.append( f'"moab@{moab}+mpi+hdf5+netcdf+pnetcdf+metis+parmetis+tempest"') if parmetis != 'None': - specs.append(f'"parmetis@{parmetis}~shared"') + specs.append(f'"parmetis@{parmetis}+int64~shared"') if petsc != 'None': specs.append(f'"petsc@{petsc}+mpi+batch"') From 53ffcf61e593be578983f7dfa06e3d4ca5e95cb9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 11 Jul 2024 08:30:28 -0500 Subject: [PATCH 3/9] Update E3SM-Project submodule to latest master --- e3sm_submodules/E3SM-Project | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e3sm_submodules/E3SM-Project b/e3sm_submodules/E3SM-Project index 1fb279e59..04842fecc 160000 --- a/e3sm_submodules/E3SM-Project +++ b/e3sm_submodules/E3SM-Project @@ -1 +1 @@ -Subproject commit 1fb279e59d0d4d9fcae708fdba80615d0c516319 +Subproject commit 04842fecc6341266ec9bf5a89dda99b9903d5b6f From 8b791ee81a0f8a4e82eef0b69131e746bc2ba093 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 11 Jul 2024 08:38:36 -0500 Subject: [PATCH 4/9] Update MALI-Dev submodule to latest develop --- e3sm_submodules/MALI-Dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e3sm_submodules/MALI-Dev b/e3sm_submodules/MALI-Dev index ace160c64..e23c40196 160000 --- a/e3sm_submodules/MALI-Dev +++ b/e3sm_submodules/MALI-Dev @@ -1 +1 @@ -Subproject commit ace160c64167a3a40d7fc9afee6c6b2efd8b6a50 +Subproject commit e23c401962f26fa2a9aee62eff3f54004aa759b0 From c96a1635c6d98e51830934f450372f695e10d1b0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 11 Jul 2024 08:39:44 -0500 Subject: [PATCH 5/9] Update Omega submodule to latest develop --- e3sm_submodules/Omega | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e3sm_submodules/Omega b/e3sm_submodules/Omega index e55160261..e9b9f4def 160000 --- a/e3sm_submodules/Omega +++ b/e3sm_submodules/Omega @@ -1 +1 @@ -Subproject commit e551602611e4afb24b208f285eee0f4c7b5bc117 +Subproject commit e9b9f4def69870b60d841fc65a298e73753f86fa From ee00889c31b23f019a90cda21313cf9fee235d0d Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 17 Jul 2024 05:19:49 -0500 Subject: [PATCH 6/9] Update to mache v1.24.0 --- deploy/default.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/default.cfg b/deploy/default.cfg index 4c0c922b2..2a3caca2f 100644 --- a/deploy/default.cfg +++ b/deploy/default.cfg @@ -21,7 +21,7 @@ mpi = nompi # versions of conda packages geometric_features = 1.2.0 -mache = 1.19.0 +mache = 1.24.0 mpas_tools = 0.34.1 otps = 2021.10 parallelio = 2.6.0 From 52e84b75b6f9ed8d8b2334285cc2731fd0e3fbaa Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 22 Jul 2024 06:50:26 -0600 Subject: [PATCH 7/9] Add spack_mirror to chicoma-cpu --- deploy/bootstrap.py | 8 +++++++- polaris/machines/chicoma-cpu.cfg | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py index e49963ba4..9121147e3 100755 --- a/deploy/bootstrap.py +++ b/deploy/bootstrap.py @@ -511,6 +511,11 @@ def build_spack_soft_env(config, update_spack, machine, env_type, # noqa: C901 esmf = config.get('deploy', 'esmf') + if config.has_option('deploy', 'spack_mirror'): + spack_mirror = config.get('deploy', 'spack_mirror') + else: + spack_mirror = None + spack_branch_base = f'{spack_base}/{spack_env}' specs = list() @@ -545,7 +550,8 @@ def build_spack_soft_env(config, update_spack, machine, env_type, # noqa: C901 spack_specs=specs, compiler=compiler, mpi=mpi, machine=machine, config_file=machine_config, include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, - yaml_template=yaml_template, tmpdir=tmpdir) + yaml_template=yaml_template, tmpdir=tmpdir, + spack_mirror=spack_mirror) spack_view = f'{spack_branch_base}/var/spack/environments/' \ f'{spack_env}/.spack-env/view' diff --git a/polaris/machines/chicoma-cpu.cfg b/polaris/machines/chicoma-cpu.cfg index 26a71e909..f22c52d2e 100644 --- a/polaris/machines/chicoma-cpu.cfg +++ b/polaris/machines/chicoma-cpu.cfg @@ -28,6 +28,9 @@ spack = /usr/projects/e3sm/polaris/chicoma-cpu/spack # pnetcdf as E3SM (spack modules are used otherwise) use_e3sm_hdf5_netcdf = True +# location of a spack mirror for polaris to use +spack_mirror = /usr/projects/e3sm/polaris/chicoma-cpu/spack/spack_mirror + # The parallel section describes options related to running jobs in parallel [parallel] From 25e91762964756d7d9772faf73d02a14814b0fe2 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 23 Jul 2024 02:55:52 -0600 Subject: [PATCH 8/9] Drop debug qos for chicoma-cpu This requires a debug reservation as well, which polaris doesn't support. --- polaris/machines/chicoma-cpu.cfg | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/polaris/machines/chicoma-cpu.cfg b/polaris/machines/chicoma-cpu.cfg index f22c52d2e..8fe0455d8 100644 --- a/polaris/machines/chicoma-cpu.cfg +++ b/polaris/machines/chicoma-cpu.cfg @@ -45,6 +45,11 @@ cores_per_node = 128 # hanging on perlmutter) threads_per_core = 1 +# quality of service +# overriding mache because the debug qos also requires --reservaiton debug, +# which polaris doesn't currently support +qos = standard + # Config options related to creating a job script [job] From f95f59e362769ad1ea73f328cdc88dda06aab7f8 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 23 Jul 2024 03:05:35 -0600 Subject: [PATCH 9/9] Add account argument to Omega CTest util --- utils/omega/ctest/omega_ctest.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/utils/omega/ctest/omega_ctest.py b/utils/omega/ctest/omega_ctest.py index f94a6b0b8..7abe4018f 100755 --- a/utils/omega/ctest/omega_ctest.py +++ b/utils/omega/ctest/omega_ctest.py @@ -13,7 +13,7 @@ def make_build_script(machine, compiler, branch, build_only, mesh_filename, planar_mesh_filename, sphere_mesh_filename, - debug, clean, cmake_flags): + debug, clean, cmake_flags, account): """ Make a shell script for checking out Omega and its submodules, building Omega and its ctests, linking to testing data files, and running ctests. @@ -45,6 +45,9 @@ def make_build_script(machine, compiler, branch, build_only, mesh_filename, if cmake_flags is None: cmake_flags = '' + if account is not None: + cmake_flags = f'{cmake_flags} -DOMEGA_CIME_PROJECT={account}' + script = template.render(update_omega_submodule=update_omega_submodule, polaris_source_dir=polaris_source_dir, omega_base_dir=branch, @@ -108,15 +111,16 @@ def download_meshes(config): return download_targets -def write_job_script(config, machine, compiler, submit): +def write_job_script(config, machine, compiler, account, submit): """ Write a job script for running the build script """ - if config.has_option('parallel', 'account'): - account = config.get('parallel', 'account') - else: - account = '' + if account is None: + if config.has_option('parallel', 'account'): + account = config.get('parallel', 'account') + else: + account = '' nodes = 1 @@ -190,6 +194,8 @@ def main(): help='Whether to only build Omega in debug mode') parser.add_argument('--cmake_flags', dest='cmake_flags', help='Quoted string with additional cmake flags') + parser.add_argument('--account', dest='account', + help='slurm account to submit the job to') args = parser.parse_args() @@ -206,6 +212,7 @@ def main(): debug = args.debug clean = args.clean cmake_flags = args.cmake_flags + account = args.account if 'SLURM_JOB_ID' in os.environ: # already on a comptue node so we will just run ctests directly @@ -221,7 +228,8 @@ def main(): build_only=build_only, mesh_filename=mesh_filename, planar_mesh_filename=planar_mesh_filename, sphere_mesh_filename=sphere_mesh_filename, - debug=debug, clean=clean, cmake_flags=cmake_flags) + debug=debug, clean=clean, cmake_flags=cmake_flags, + account=account) # clear environment variables and start fresh with those from login # so spack doesn't get confused by conda @@ -229,7 +237,7 @@ def main(): shell=True) write_job_script(config=config, machine=machine, compiler=compiler, - submit=submit) + account=account, submit=submit) if __name__ == '__main__':