From b2ca9eb6a5c33901e5cd4e02524010d2e8bb3d9f Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 15 Jul 2024 07:34:53 -0500 Subject: [PATCH 1/6] Add moab with tempest to conda environment --- deploy/bootstrap.py | 2 +- deploy/conda-dev-spec.template | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py index 89931c70d..852519fb1 100755 --- a/deploy/bootstrap.py +++ b/deploy/bootstrap.py @@ -291,7 +291,7 @@ def build_conda_env(config, env_type, recreate, mpi, conda_mpi, version, mpi_prefix=mpi_prefix, include_mache=not local_mache) - for package in ['esmf', 'geometric_features', 'mache', 'metis', + for package in ['esmf', 'geometric_features', 'mache', 'metis', 'moab', 'mpas_tools', 'netcdf_c', 'netcdf_fortran', 'otps', 'parallelio', 'pnetcdf']: replacements[package] = config.get('deploy', package) diff --git a/deploy/conda-dev-spec.template b/deploy/conda-dev-spec.template index 5ec77dc17..d22439ea8 100644 --- a/deploy/conda-dev-spec.template +++ b/deploy/conda-dev-spec.template @@ -23,6 +23,7 @@ mache={{ mache }} {% endif %} matplotlib-base>=3.9.0 metis={{ metis }} +moab={{ moab }}=*_tempest_* mpas_tools={{ mpas_tools }} nco netcdf4=*=nompi_* From 83c286f035757c1137f54ab0cabb498e08e6451e Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 15 Jul 2024 08:23:15 -0500 Subject: [PATCH 2/6] Allow multiple commands in Step's args attribute This will be needed for `mbtempest` support and may be useful in other circumstances. --- polaris/model_step.py | 4 ++-- polaris/run/serial.py | 9 +++++---- polaris/step.py | 6 ++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/polaris/model_step.py b/polaris/model_step.py index 8eb6c3614..b0a6dc2a1 100644 --- a/polaris/model_step.py +++ b/polaris/model_step.py @@ -177,8 +177,8 @@ def setup(self): config = self.config component_path = config.get('executables', 'component') model_basename = os.path.basename(component_path) - self.args = [f'./{model_basename}', '-n', self.namelist, - '-s', self.streams] + self.args = [[f'./{model_basename}', '-n', self.namelist, + '-s', self.streams]] def set_model_resources(self, ntasks=None, min_tasks=None, openmp_threads=None, max_memory=None): diff --git a/polaris/run/serial.py b/polaris/run/serial.py index feb07c3ce..63ba63705 100644 --- a/polaris/run/serial.py +++ b/polaris/run/serial.py @@ -490,10 +490,11 @@ def _run_step(task, step, new_log_file, available_resources, if step.args is not None: step_logger.info('\nBypassing step\'s run() method and running ' 'with command line args\n') - log_function_call(function=run_command, logger=step_logger) - step_logger.info('') - run_command(step.args, step.cpus_per_task, step.ntasks, - step.openmp_threads, step.config, step.logger) + for args in step.args: + log_function_call(function=run_command, logger=step_logger) + step_logger.info('') + run_command(args, step.cpus_per_task, step.ntasks, + step.openmp_threads, step.config, step.logger) else: step_logger.info('') log_method_call(method=step.run, logger=step_logger) diff --git a/polaris/step.py b/polaris/step.py index 9e92a0809..146e97c34 100644 --- a/polaris/step.py +++ b/polaris/step.py @@ -142,8 +142,10 @@ class Step: file (e.g. if the step calls external code that, in turn, calls additional subprocesses). - args : {list of str, None} - A list of command-line arguments to call in parallel + args : {list of list of str, None} + A list of lists of command-line arguments to call in parallel. Each + inner list represents a single command. All commands must use the + same parallel resources. """ def __init__(self, component, name, subdir=None, indir=None, From efc34b3a9df3aaa09452c2694acbe07ae51e9c7c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 15 Jul 2024 08:36:05 -0500 Subject: [PATCH 3/6] Add remap_tool config option and reorganize For now, esmf is the only supported remap tool. --- polaris/remap/mapping_file_step.py | 65 ++++++++++++++++++------------ polaris/remap/remap.cfg | 5 +++ 2 files changed, 44 insertions(+), 26 deletions(-) create mode 100644 polaris/remap/remap.cfg diff --git a/polaris/remap/mapping_file_step.py b/polaris/remap/mapping_file_step.py index 1b86dcbe2..6ae3c635a 100644 --- a/polaris/remap/mapping_file_step.py +++ b/polaris/remap/mapping_file_step.py @@ -381,21 +381,36 @@ def runtime_setup(self): Create a remapper and set the command-line arguments """ remapper = self.get_remapper() - self.args = _build_mapping_file_args(remapper, self.method, - self.expand_distance, - self.expand_factor, - self.src_mesh_filename, - self.dst_mesh_filename) + remap_tool = self.config.get('remap', 'remap_tool') + _check_remapper(remapper, self.method, remap_tool=remap_tool) + if remap_tool == 'esmf': + self.args = [_esmf_build_map_args(remapper, self.method, + self.expand_distance, + self.expand_factor, + self.src_mesh_filename, + self.dst_mesh_filename)] -def _build_mapping_file_args(remapper, method, expand_distance, expand_factor, - src_mesh_filename, dst_mesh_filename): + +def _check_remapper(remapper, method, remap_tool): """ - Get command-line arguments for making a mapping file + Check for inconsistencies in the remapper """ + if isinstance(remapper.destinationDescriptor, + PointCollectionDescriptor) and \ + method not in ['bilinear', 'neareststod']: + raise ValueError(f'method {method} not supported for destination ' + 'grid of type PointCollectionDescriptor.') - _check_remapper(remapper, method) + if remap_tool == 'moab' and method == 'neareststod': + raise ValueError('method neareststod not supported by mbtempest.') + +def _esmf_build_map_args(remapper, method, expand_distance, expand_factor, + src_mesh_filename, dst_mesh_filename): + """ + Get command-line arguments for making a mapping file + """ src_descriptor = remapper.sourceDescriptor src_descriptor.to_scrip(src_mesh_filename) @@ -408,8 +423,7 @@ def _build_mapping_file_args(remapper, method, expand_distance, expand_factor, '--destination', dst_mesh_filename, '--weight', remapper.mappingFileName, '--method', method, - '--netcdf4', - '--no_log'] + '--netcdf4'] if src_descriptor.regional: args.append('--src_regional') @@ -423,19 +437,10 @@ def _build_mapping_file_args(remapper, method, expand_distance, expand_factor, return args -def _check_remapper(remapper, method): +def _get_descriptor(info): """ - Check for inconsistencies in the remapper + Get a mesh descriptor from the mesh info """ - if isinstance(remapper.destinationDescriptor, - PointCollectionDescriptor) and \ - method not in ['bilinear', 'neareststod']: - raise ValueError(f'method {method} not supported for destination ' - 'grid of type PointCollectionDescriptor.') - - -def _get_descriptor(info): - """ Get a mesh descriptor from the mesh info """ grid_type = info['type'] if grid_type == 'mpas': descriptor = _get_mpas_descriptor(info) @@ -451,7 +456,9 @@ def _get_descriptor(info): def _get_mpas_descriptor(info): - """ Get an MPAS mesh descriptor from the given info """ + """ + Get an MPAS mesh descriptor from the given info + """ mesh_type = info['mpas_mesh_type'] filename = info['filename'] mesh_name = info['name'] @@ -472,7 +479,9 @@ def _get_mpas_descriptor(info): def _get_lon_lat_descriptor(info): - """ Get a lon-lat descriptor from the given info """ + """ + Get a lon-lat descriptor from the given info + """ if 'dlat' in info and 'dlon' in info: lon_min = info['lon_min'] @@ -510,7 +519,9 @@ def _get_lon_lat_descriptor(info): def _get_proj_descriptor(info): - """ Get a ProjectionGridDescriptor from the given info """ + """ + Get a ProjectionGridDescriptor from the given info + """ filename = info['filename'] grid_name = info['name'] x = info['x'] @@ -533,7 +544,9 @@ def _get_proj_descriptor(info): def _get_points_descriptor(info): - """ Get a PointCollectionDescriptor from the given info """ + """ + Get a PointCollectionDescriptor from the given info + """ filename = info['filename'] collection_name = info['name'] lon_var = info['lon'] diff --git a/polaris/remap/remap.cfg b/polaris/remap/remap.cfg new file mode 100644 index 000000000..75fe3bd7e --- /dev/null +++ b/polaris/remap/remap.cfg @@ -0,0 +1,5 @@ +# config options related to creating mapping files and remapping data +[remap] + +# The tool to use for remapping: esmf or moab +remap_tool = esmf From 1a8968edefb93f7326a750f8623fecb9908cb9ee Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 16 Jul 2024 09:11:50 -0500 Subject: [PATCH 4/6] Add support for making mapping files with moab --- polaris/remap/mapping.cfg | 5 ++ polaris/remap/mapping_file_step.py | 124 +++++++++++++++++++++++------ polaris/remap/remap.cfg | 5 -- 3 files changed, 104 insertions(+), 30 deletions(-) create mode 100644 polaris/remap/mapping.cfg delete mode 100644 polaris/remap/remap.cfg diff --git a/polaris/remap/mapping.cfg b/polaris/remap/mapping.cfg new file mode 100644 index 000000000..a4f707d5a --- /dev/null +++ b/polaris/remap/mapping.cfg @@ -0,0 +1,5 @@ +# config options related to creating mapping files +[mapping] + +# The tool to use for creating mapping files: esmf or moab +map_tool = moab diff --git a/polaris/remap/mapping_file_step.py b/polaris/remap/mapping_file_step.py index 6ae3c635a..d16762aaa 100644 --- a/polaris/remap/mapping_file_step.py +++ b/polaris/remap/mapping_file_step.py @@ -366,9 +366,21 @@ def get_remapper(self): out_descriptor = _get_descriptor(dst) if self.map_filename is None: + map_tool = self.config.get('mapping', 'map_tool') + prefixes = { + 'esmf': 'esmf', + 'moab': 'mbtr' + } + suffixes = { + 'conserve': 'aave', + 'bilinear': 'bilin', + 'neareststod': 'neareststod' + } + suffix = f'{prefixes[map_tool]}{suffixes[self.method]}' + self.map_filename = \ f'map_{in_descriptor.meshName}_to_{out_descriptor.meshName}' \ - f'_{self.method}.nc' + f'_{suffix}.nc' self.map_filename = os.path.abspath(os.path.join( self.work_dir, self.map_filename)) @@ -381,42 +393,55 @@ def runtime_setup(self): Create a remapper and set the command-line arguments """ remapper = self.get_remapper() - remap_tool = self.config.get('remap', 'remap_tool') - _check_remapper(remapper, self.method, remap_tool=remap_tool) - - if remap_tool == 'esmf': - self.args = [_esmf_build_map_args(remapper, self.method, - self.expand_distance, - self.expand_factor, - self.src_mesh_filename, - self.dst_mesh_filename)] - - -def _check_remapper(remapper, method, remap_tool): + map_tool = self.config.get('mapping', 'map_tool') + _check_remapper(remapper, self.method, map_tool=map_tool) + + src_descriptor = remapper.sourceDescriptor + src_descriptor.to_scrip(self.src_mesh_filename) + + dst_descriptor = remapper.destinationDescriptor + dst_descriptor.to_scrip(self.dst_mesh_filename, + expandDist=self.expand_distance, + expandFactor=self.expand_factor) + + if map_tool == 'esmf': + self.args = _esmf_build_map_args(remapper, self.method, + src_descriptor, + self.src_mesh_filename, + dst_descriptor, + self.dst_mesh_filename) + elif map_tool == 'moab': + self.args = _moab_build_map_args(remapper, self.method, + src_descriptor, + self.src_mesh_filename, + dst_descriptor, + self.dst_mesh_filename) + + +def _check_remapper(remapper, method, map_tool): """ Check for inconsistencies in the remapper """ + if map_tool not in ['moab', 'esmf']: + raise ValueError(f'Unexpected map_tool {map_tool}. Valid ' + f'values are "esmf" or "moab".') + if isinstance(remapper.destinationDescriptor, PointCollectionDescriptor) and \ method not in ['bilinear', 'neareststod']: raise ValueError(f'method {method} not supported for destination ' - 'grid of type PointCollectionDescriptor.') + f'grid of type PointCollectionDescriptor.') - if remap_tool == 'moab' and method == 'neareststod': + if map_tool == 'moab' and method == 'neareststod': raise ValueError('method neareststod not supported by mbtempest.') -def _esmf_build_map_args(remapper, method, expand_distance, expand_factor, - src_mesh_filename, dst_mesh_filename): +def _esmf_build_map_args(remapper, method, src_descriptor, src_mesh_filename, + dst_descriptor, dst_mesh_filename): """ - Get command-line arguments for making a mapping file + Get command-line arguments for making a mapping file with + ESMF_RegridWeightGen """ - src_descriptor = remapper.sourceDescriptor - src_descriptor.to_scrip(src_mesh_filename) - - dst_descriptor = remapper.destinationDescriptor - dst_descriptor.to_scrip(dst_mesh_filename, expandDist=expand_distance, - expandFactor=expand_factor) args = ['ESMF_RegridWeightGen', '--source', src_mesh_filename, @@ -434,7 +459,53 @@ def _esmf_build_map_args(remapper, method, expand_distance, expand_factor, if src_descriptor.regional or dst_descriptor.regional: args.append('--ignore_unmapped') - return args + return [args] + + +def _moab_build_map_args(remapper, method, src_descriptor, src_mesh_filename, + dst_descriptor, dst_mesh_filename): + """ + Get command-line arguments for making a mapping file with mbtempest + """ + fvmethod = { + 'conserve': 'none', + 'bilinear': 'bilin'} + + map_filename = remapper.mappingFileName + intx_filename = \ + f'moab_intx_{src_descriptor.meshName}_to_{dst_descriptor.meshName}.h5m' + + intx_args = [ + 'mbtempest', + '--type', '5', + '--load', src_mesh_filename, + '--load', dst_mesh_filename, + '--intx', intx_filename + ] + + if src_descriptor.regional or dst_descriptor.regional: + intx_args.append('--rrmgrids') + + map_args = [ + 'mbtempest', + '--type', '5', + '--load', src_mesh_filename, + '--load', dst_mesh_filename, + '--intx', intx_filename, + '--weights', + '--method', 'fv', + '--method', 'fv', + '--file', map_filename, + '--order', '1', + '--order', '1', + '--fvmethod', fvmethod[method] + ] + + if method == 'conserve' and (src_descriptor.regional or + dst_descriptor.regional): + map_args.append('--rrmgrids') + + return [intx_args, map_args] def _get_descriptor(info): @@ -452,6 +523,9 @@ def _get_descriptor(info): descriptor = _get_points_descriptor(info) else: raise ValueError(f'Unexpected grid type {grid_type}') + + # for compatibility with mbtempest + descriptor.format = 'NETCDF3_64BIT_DATA' return descriptor diff --git a/polaris/remap/remap.cfg b/polaris/remap/remap.cfg deleted file mode 100644 index 75fe3bd7e..000000000 --- a/polaris/remap/remap.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# config options related to creating mapping files and remapping data -[remap] - -# The tool to use for remapping: esmf or moab -remap_tool = esmf From 41532a5998695551892a8f8105b90857ab9b64c0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 15 Jul 2024 08:38:02 -0500 Subject: [PATCH 5/6] Update isomip_plus to include mapping config options --- polaris/ocean/tasks/isomip_plus/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/polaris/ocean/tasks/isomip_plus/__init__.py b/polaris/ocean/tasks/isomip_plus/__init__.py index c18e6bcff..7e5cd7f2a 100644 --- a/polaris/ocean/tasks/isomip_plus/__init__.py +++ b/polaris/ocean/tasks/isomip_plus/__init__.py @@ -34,6 +34,8 @@ def add_isomip_plus_tasks(component, mesh_type): config.set('spherical_mesh', 'mpas_mesh_filename', 'base_mesh_without_xy.nc') + config.add_from_package('polaris.remap', 'mapping.cfg') + config.add_from_package('polaris.ocean.tasks.isomip_plus', 'isomip_plus.cfg') From fadf5c2bfd5908ab03c7a3b5a902e4eecec8ebef Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 17 Jul 2024 04:10:35 -0500 Subject: [PATCH 6/6] Update the docs --- docs/developers_guide/framework/remapping.md | 44 +++++++++++++++----- docs/developers_guide/quick_start.md | 3 +- docs/developers_guide/troubleshooting.md | 2 +- docs/users_guide/quick_start.md | 3 +- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/docs/developers_guide/framework/remapping.md b/docs/developers_guide/framework/remapping.md index bf9ba3a80..272cda329 100644 --- a/docs/developers_guide/framework/remapping.md +++ b/docs/developers_guide/framework/remapping.md @@ -6,17 +6,18 @@ It is frequently useful when working with observational datasets or visualizing MPAS data to remap between different global or regional grids and meshes. The [pyremap](https://mpas-dev.github.io/pyremap/stable/) provides capabilities for making mapping files (which contain the weights needed to -interpolate between meshes) and using them to remap files or +interpolate between meshes) and using them to remap files or {py:class}`xarray.Dataset` objects. Polaris provides a step for producing such a mapping file. Under the hood, `pyremap` uses the [ESMF_RegridWeightGen](https://earthsystemmodeling.org/docs/release/latest/ESMF_refdoc/node3.html#SECTION03020000000000000000) -tool, which uses MPI parallelism. To better support task parallelism, it is +or [mbtempest](https://sigma.mcs.anl.gov/moab/offline-remapping-workflow-with-mbtempest/) +tools, which use MPI parallelism. To better support task parallelism, it is best to have each MPI task be a separate polaris step. For this reason, we provide {py:class}`polaris.remap.MappingFileStep` to perform remapping. A remapping step can be added to a task either by creating a {py:class}`polaris.remap.MappingFileStep` object directly or by creating a -step that descends from the class. Here is an example of using +step that descends from the class. Here is an example of using `MappingFileStep` directly to remap data from a WOA 2023 lon-lat grid to an MPAS mesh. This could happen in the task's `__init__()` or `configure()` method: @@ -24,12 +25,18 @@ method: ```python from polaris import Task +from polaris.config import PolarisConfigParser from polaris.remap import MappingFileStep class MyTestCase(Task): def __int__(self, component): - step = MappingFileStep(component=component, name='make_map', ntasks=64, + step = MappingFileStep(component=component, name='make_map', ntasks=64, min_tasks=1, method='bilinear') + # add required config options related to mapping + config = PolarisConfigParser(filepath=filepath) + config.add_from_package('polaris.remap', 'mapping.cfg') + step.set_shared_config(config, link='my_test_case.cfg') + # indicate the the mesh from another step is an input to this step # note: the target is relative to the step, not the task. step.add_input_file(filename='woa.nc', @@ -38,7 +45,7 @@ class MyTestCase(Task): step.add_input_file(filename='mesh.nc', target='../mesh/mesh.nc') - + # you need to specify what type of source and destination mesh you # will use step.src_from_lon_lat(filename='woa.nc', lon_var='lon', lat_var='lat') @@ -62,7 +69,7 @@ from polaris.remap import MappingFileStep class VizMap(MappingFileStep): - def __init__(self, component, name, subdir, mesh_name): + def __init__(self, component, name, subdir, mesh_name, config): super().__init__(component=component, name=name, subdir=subdir, ntasks=128, min_tasks=1) self.mesh_name = mesh_name @@ -81,11 +88,26 @@ class VizMap(MappingFileStep): super().runtime_setup() ``` -With either approach, you will need to call one of the `src_*()` methods to -set up the source mesh or grid and one of the `dst_*()` to configure the -destination. Expect for lon-lat grids, you will need to provide a name for -the mesh or grid, typically describing its resolution and perhaps its extent -and the region covered. +It is important that the task that the step belongs to includes the required +config options related to mapping. This could be accomplished either by +calling: +```python +config.add_from_package('polaris.remap', 'mapping.cfg') +``` +or by including the corresponding config options in the task's config file: +```cfg +# config options related to creating mapping files +[mapping] + +# The tool to use for creating mapping files: esmf or moab +map_tool = moab +``` + +Whether you create a `MappingFileStep` object directly or create a subclass, +you will need to call one of the `src_*()` methods to set up the source mesh or +grid and one of the `dst_*()` to configure the destination. Expect for lon-lat +grids, you will need to provide a name for the mesh or grid, typically +describing its resolution and perhaps its extent and the region covered. In nearly all situations, creating the mapping file is only one step in the workflow. After that, the mapping file will be used to remap data between diff --git a/docs/developers_guide/quick_start.md b/docs/developers_guide/quick_start.md index ebf155761..0d72fa99d 100644 --- a/docs/developers_guide/quick_start.md +++ b/docs/developers_guide/quick_start.md @@ -182,7 +182,8 @@ this script will also: build several libraries with system compilers and MPI library, including: [SCORPIO](https://github.com/E3SM-Project/scorpio) (parallel i/o for E3SM components) [ESMF](https://earthsystemmodeling.org/) (making mapping files - in parallel), [Trilinos](https://trilinos.github.io/), + in parallel), [MOAB](https://sigma.mcs.anl.gov/moab-library/), + [Trilinos](https://trilinos.github.io/), [Albany](https://github.com/sandialabs/Albany), [Netlib-LAPACK](http://www.netlib.org/lapack/) and [PETSc](https://petsc.org/). **Please uses these flags with caution, as diff --git a/docs/developers_guide/troubleshooting.md b/docs/developers_guide/troubleshooting.md index 39e6ae306..1e596caf6 100644 --- a/docs/developers_guide/troubleshooting.md +++ b/docs/developers_guide/troubleshooting.md @@ -117,5 +117,5 @@ of your user config file: parallel_executable = mpirun -host localhost ``` -The [example config file](https://github.com/E3SM-Project/polaris/blob/main/exmaple.cfg) +The [example config file](https://github.com/E3SM-Project/polaris/blob/main/example.cfg) has been updated to include this flag. diff --git a/docs/users_guide/quick_start.md b/docs/users_guide/quick_start.md index e896f21d7..38d5abbe3 100644 --- a/docs/users_guide/quick_start.md +++ b/docs/users_guide/quick_start.md @@ -17,7 +17,8 @@ documentation as soon as there is one. Until then please refer to the For each polaris release, we maintain a [conda environment](https://docs.conda.io/en/latest/). that includes the `polaris` package as well as all of its dependencies and some libraries -(currently [ESMF](https://earthsystemmodeling.org/) and +(currently [ESMF](https://earthsystemmodeling.org/), +[MOAB](https://sigma.mcs.anl.gov/moab-library/) and [SCORPIO](https://e3sm.org/scorpio-parallel-io-library/)) built with system MPI using [spack](https://spack.io/) on our standard machines (Anvil, Chicoma, Chrysalis, Compy, and Perlmutter). Once there is a polaris release,