From 711208110eb9ccd9964c1f6d665571c2024317dc Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 21 Sep 2023 15:33:22 +0200 Subject: [PATCH 1/4] Do not add config options for tasks automatically The developer can lose control of whether config options take precedence over others because the file get added before the `configure()` method is called. --- polaris/setup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/polaris/setup.py b/polaris/setup.py index f82bfefb5..0bb3edfe7 100644 --- a/polaris/setup.py +++ b/polaris/setup.py @@ -189,10 +189,6 @@ def setup_task(path, task, config_file, machine, work_dir, baseline_dir, if copy_executable: config.set('setup', 'copy_executable', 'True') - # add the config options for the task (if defined) - config.add_from_package(task.__module__, - f'{task.name}.cfg', exception=False) - if 'POLARIS_BRANCH' in os.environ: polaris_branch = os.environ['POLARIS_BRANCH'] config.set('paths', 'polaris_branch', polaris_branch) From 66ea628c0b91f4f7e6c65a3dda19c67a6c5b59b4 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 21 Sep 2023 09:30:26 +0200 Subject: [PATCH 2/4] Add ocean framework for spherical convergence tests --- polaris/ocean/convergence/__init__.py | 0 .../ocean/convergence/spherical/__init__.py | 3 + .../ocean/convergence/spherical/forward.py | 145 ++++++++++++++++++ .../ocean/convergence/spherical/spherical.cfg | 34 ++++ polaris/ocean/model/__init__.py | 1 + polaris/ocean/model/time.py | 36 +++++ 6 files changed, 219 insertions(+) create mode 100644 polaris/ocean/convergence/__init__.py create mode 100644 polaris/ocean/convergence/spherical/__init__.py create mode 100644 polaris/ocean/convergence/spherical/forward.py create mode 100644 polaris/ocean/convergence/spherical/spherical.cfg create mode 100644 polaris/ocean/model/time.py diff --git a/polaris/ocean/convergence/__init__.py b/polaris/ocean/convergence/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/polaris/ocean/convergence/spherical/__init__.py b/polaris/ocean/convergence/spherical/__init__.py new file mode 100644 index 000000000..702a1352f --- /dev/null +++ b/polaris/ocean/convergence/spherical/__init__.py @@ -0,0 +1,3 @@ +from polaris.ocean.convergence.spherical.forward import ( + SphericalConvergenceForward, +) diff --git a/polaris/ocean/convergence/spherical/forward.py b/polaris/ocean/convergence/spherical/forward.py new file mode 100644 index 000000000..aa14f26d8 --- /dev/null +++ b/polaris/ocean/convergence/spherical/forward.py @@ -0,0 +1,145 @@ +from polaris.ocean.model import OceanModelStep, get_time_interval_string + + +class SphericalConvergenceForward(OceanModelStep): + """ + A step for performing forward ocean component runs as part of a spherical + convergence test + + Attributes + ---------- + resolution : float + The resolution of the (uniform) mesh in km + + package : Package + The package name or module object that contains the YAML file + + yaml_filename : str + A YAML file that is a Jinja2 template with model config options + + """ + + def __init__(self, component, name, subdir, resolution, base_mesh, init, + package, yaml_filename='forward.yaml', options=None, + output_filename='output.nc', validate_vars=None): + """ + Create a new step + + Parameters + ---------- + component : polaris.Component + The component the step belongs to + + name : str + The name of the step + + subdir : str + The subdirectory for the step + + resolution : float + The resolution of the (uniform) mesh in km + + package : Package + The package name or module object that contains the YAML file + + yaml_filename : str, optional + A YAML file that is a Jinja2 template with model config options + + options : dict, optional + A dictionary of options and value to replace model config options + with new values + + output_filename : str, optional + The output file that will be written out at the end of the forward + run + + validate_vars : list of str, optional + A list of variables to validate against a baseline if requested + """ + super().__init__(component=component, name=name, subdir=subdir, + openmp_threads=1) + + self.resolution = resolution + self.package = package + self.yaml_filename = yaml_filename + + # make sure output is double precision + self.add_yaml_file('polaris.ocean.config', 'output.yaml') + + if options is not None: + self.add_model_config_options(options=options) + + self.add_input_file( + filename='init.nc', + work_dir_target=f'{init.path}/initial_state.nc') + self.add_input_file( + filename='graph.info', + work_dir_target=f'{base_mesh.path}/graph.info') + + self.add_output_file(filename=output_filename, + validate_vars=validate_vars) + + def compute_cell_count(self): + """ + Compute the approximate number of cells in the mesh, used to constrain + resources + + Returns + ------- + cell_count : int or None + The approximate number of cells in the mesh + """ + # use a heuristic based on QU30 (65275 cells) and QU240 (10383 cells) + cell_count = 6e8 / self.resolution**2 + return cell_count + + def dynamic_model_config(self, at_setup): + """ + Set the model time step from config options at setup and runtime + + Parameters + ---------- + at_setup : bool + Whether this method is being run during setup of the step, as + opposed to at runtime + """ + super().dynamic_model_config(at_setup=at_setup) + + config = self.config + + vert_levels = config.getfloat('vertical_grid', 'vert_levels') + if not at_setup and vert_levels == 1: + self.add_yaml_file('polaris.ocean.config', 'single_layer.yaml') + + section = config['spherical_convergence_forward'] + + time_integrator = section.get('time_integrator') + + # dt is proportional to resolution: default 30 seconds per km + if time_integrator == 'RK4': + dt_per_km = section.getfloat('rk4_dt_per_km') + else: + dt_per_km = section.getfloat('split_dt_per_km') + dt_str = get_time_interval_string(seconds=dt_per_km * self.resolution) + + # btr_dt is also proportional to resolution: default 1.5 seconds per km + btr_dt_per_km = section.getfloat('btr_dt_per_km') + btr_dt_str = get_time_interval_string( + seconds=btr_dt_per_km * self.resolution) + + run_duration = section.getfloat('run_duration') + run_duration_str = get_time_interval_string(days=run_duration) + + output_interval = section.getfloat('output_interval') + output_interval_str = get_time_interval_string(days=output_interval) + + replacements = dict( + time_integrator=time_integrator, + dt=dt_str, + btr_dt=btr_dt_str, + run_duration=run_duration_str, + output_interval=output_interval_str, + ) + + self.add_yaml_file(self.package, self.yaml_filename, + template_replacements=replacements) diff --git a/polaris/ocean/convergence/spherical/spherical.cfg b/polaris/ocean/convergence/spherical/spherical.cfg new file mode 100644 index 000000000..5b2e00269 --- /dev/null +++ b/polaris/ocean/convergence/spherical/spherical.cfg @@ -0,0 +1,34 @@ +# config options for spherical convergence tests +[spherical_convergence] + +# a list of icosahedral mesh resolutions (km) to test +icos_resolutions = 60, 120, 240, 480 + +# a list of quasi-uniform mesh resolutions (km) to test +qu_resolutions = 60, 90, 120, 150, 180, 210, 240 + +# Evaluation time for convergence analysis (in days) +convergence_eval_time = 1.0 + + +# config options for spherical convergence forward steps +[spherical_convergence_forward] + +# time integrator: {'split_explicit', 'RK4'} +time_integrator = RK4 + +# RK4 time step per resolution (s/km), since dt is proportional to resolution +rk4_dt_per_km = 3.0 + +# split time step per resolution (s/km), since dt is proportional to resolution +split_dt_per_km = 30.0 + +# the barotropic time step (s/km) for simulations using split time stepping, +# since btr_dt is proportional to resolution +btr_dt_per_km = 1.5 + +# Run duration in days +run_duration = ${spherical_convergence:convergence_eval_time} + +# Output interval in days +output_interval = ${run_duration} diff --git a/polaris/ocean/model/__init__.py b/polaris/ocean/model/__init__.py index f4be24f8f..3bcb9b7f4 100644 --- a/polaris/ocean/model/__init__.py +++ b/polaris/ocean/model/__init__.py @@ -1 +1,2 @@ from polaris.ocean.model.ocean_model_step import OceanModelStep +from polaris.ocean.model.time import get_time_interval_string diff --git a/polaris/ocean/model/time.py b/polaris/ocean/model/time.py new file mode 100644 index 000000000..84e0e941b --- /dev/null +++ b/polaris/ocean/model/time.py @@ -0,0 +1,36 @@ +import time + + +def get_time_interval_string(days=None, seconds=None): + """ + Convert a time interval in days and/or seconds to a string for use in a + model config option. If both are provided, they will be added + + Parameters + ---------- + days : float, optional + A time interval in days + + seconds : float, optional + A time interval in seconds + + Returns + ------- + time_str : str + The time as a string in the format "DDDD_HH:MM:SS.SS" + + """ + sec_per_day = 86400 + total = 0. + if seconds is not None: + total += seconds + if days is not None: + total += sec_per_day * days + + day_part = int(total / sec_per_day) + sec_part = total - day_part * sec_per_day + + # https://stackoverflow.com/a/1384565/7728169 + seconds_str = time.strftime('%H:%M:%S', time.gmtime(sec_part)) + time_str = f'{day_part:04d}_{seconds_str}' + return time_str From 5bc60ad973dbc7f103ff5afa0eaf9995b416f59f Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 21 Sep 2023 09:33:13 +0200 Subject: [PATCH 3/4] Update cosine bell to use shared convergence framework --- polaris/ocean/tasks/cosine_bell/__init__.py | 76 ++++++++------- polaris/ocean/tasks/cosine_bell/analysis.py | 33 ++++--- .../ocean/tasks/cosine_bell/cosine_bell.cfg | 27 +++++- polaris/ocean/tasks/cosine_bell/forward.py | 97 +++---------------- polaris/ocean/tasks/cosine_bell/forward.yaml | 7 +- polaris/ocean/tasks/cosine_bell/init.py | 10 +- polaris/ocean/tasks/cosine_bell/viz.py | 30 ++++-- 7 files changed, 132 insertions(+), 148 deletions(-) diff --git a/polaris/ocean/tasks/cosine_bell/__init__.py b/polaris/ocean/tasks/cosine_bell/__init__.py index db90f1183..a3cee04b4 100644 --- a/polaris/ocean/tasks/cosine_bell/__init__.py +++ b/polaris/ocean/tasks/cosine_bell/__init__.py @@ -1,4 +1,6 @@ -from polaris import Task +from typing import Dict + +from polaris import Step, Task from polaris.config import PolarisConfigParser from polaris.ocean.mesh.spherical import add_spherical_base_mesh_step from polaris.ocean.tasks.cosine_bell.analysis import Analysis @@ -24,11 +26,11 @@ def add_cosine_bell_tasks(component): class CosineBell(Task): """ - A test case for creating a global MPAS-Ocean mesh + A convergence test for the advection of a cosine-bell tracer Attributes ---------- - resolutions : list of int + resolutions : list of float A list of mesh resolutions icosahedral : bool @@ -39,7 +41,7 @@ class CosineBell(Task): """ def __init__(self, component, icosahedral, include_viz): """ - Create test case for creating a global MPAS-Ocean mesh + Create the convergence test Parameters ---------- @@ -67,8 +69,8 @@ def __init__(self, component, icosahedral, include_viz): # add the steps with default resolutions so they can be listed config = PolarisConfigParser() - package = 'polaris.ocean.tasks.cosine_bell' - config.add_from_package(package, 'cosine_bell.cfg') + config.add_from_package('polaris.ocean.convergence.spherical', + 'spherical.cfg') self._setup_steps(config) def configure(self): @@ -78,6 +80,8 @@ def configure(self): super().configure() config = self.config config.add_from_package('polaris.mesh', 'mesh.cfg') + config.add_from_package('polaris.ocean.convergence.spherical', + 'spherical.cfg') config.add_from_package('polaris.ocean.tasks.cosine_bell', 'cosine_bell.cfg') @@ -86,18 +90,14 @@ def configure(self): def _setup_steps(self, config): """ setup steps given resolutions """ - if self.icosahedral: - default_resolutions = '60, 120, 240, 480' + icosahedral = self.icosahedral + if icosahedral: + prefix = 'icos' else: - default_resolutions = '60, 90, 120, 150, 180, 210, 240' - - # set the default values that a user may change before setup - config.set('cosine_bell', 'resolutions', default_resolutions, - comment='a list of resolutions (km) to test') + prefix = 'qu' - # get the resolutions back, perhaps with values set in the user's - # config file, which takes priority over what we just set above - resolutions = config.getlist('cosine_bell', 'resolutions', dtype=int) + resolutions = config.getlist('spherical_convergence', + f'{prefix}_resolutions', dtype=float) if self.resolutions == resolutions: return @@ -109,16 +109,14 @@ def _setup_steps(self, config): self.resolutions = resolutions component = self.component - icosahedral = self.icosahedral - if icosahedral: - prefix = 'icos' - else: - prefix = 'qu' + analysis_dependencies: Dict[str, Dict[str, Step]] = ( + dict(mesh=dict(), init=dict(), forward=dict())) for resolution in resolutions: - base_mesh, mesh_name = add_spherical_base_mesh_step( + base_mesh_step, mesh_name = add_spherical_base_mesh_step( component, resolution, icosahedral) - self.add_step(base_mesh, symlink=f'base_mesh/{mesh_name}') + self.add_step(base_mesh_step, symlink=f'base_mesh/{mesh_name}') + analysis_dependencies['mesh'][resolution] = base_mesh_step cos_bell_dir = f'spherical/{prefix}/cosine_bell' @@ -129,11 +127,12 @@ def _setup_steps(self, config): else: symlink = None if subdir in component.steps: - step = component.steps[subdir] + init_step = component.steps[subdir] else: - step = Init(component=component, name=name, subdir=subdir, - mesh_name=mesh_name) - self.add_step(step, symlink=symlink) + init_step = Init(component=component, name=name, subdir=subdir, + base_mesh=base_mesh_step) + self.add_step(init_step, symlink=symlink) + analysis_dependencies['init'][resolution] = init_step name = f'{prefix}_forward_{mesh_name}' subdir = f'{cos_bell_dir}/forward/{mesh_name}' @@ -142,12 +141,14 @@ def _setup_steps(self, config): else: symlink = None if subdir in component.steps: - step = component.steps[subdir] + forward_step = component.steps[subdir] else: - step = Forward(component=component, name=name, - subdir=subdir, resolution=resolution, - mesh_name=mesh_name) - self.add_step(step, symlink=symlink) + forward_step = Forward(component=component, name=name, + subdir=subdir, resolution=resolution, + base_mesh=base_mesh_step, + init=init_step) + self.add_step(forward_step, symlink=symlink) + analysis_dependencies['forward'][resolution] = forward_step if self.include_viz: with_viz_dir = f'spherical/{prefix}/cosine_bell/with_viz' @@ -155,14 +156,16 @@ def _setup_steps(self, config): name = f'{prefix}_map_{mesh_name}' subdir = f'{with_viz_dir}/map/{mesh_name}' viz_map = VizMap(component=component, name=name, - subdir=subdir, mesh_name=mesh_name) + subdir=subdir, base_mesh=base_mesh_step, + mesh_name=mesh_name) self.add_step(viz_map) name = f'{prefix}_viz_{mesh_name}' subdir = f'{with_viz_dir}/viz/{mesh_name}' step = Viz(component=component, name=name, - subdir=subdir, viz_map=viz_map, - mesh_name=mesh_name) + subdir=subdir, base_mesh=base_mesh_step, + init=init_step, forward=forward_step, + viz_map=viz_map, mesh_name=mesh_name) self.add_step(step) subdir = f'spherical/{prefix}/cosine_bell/analysis' @@ -174,5 +177,6 @@ def _setup_steps(self, config): step = component.steps[subdir] else: step = Analysis(component=component, resolutions=resolutions, - icosahedral=icosahedral, subdir=subdir) + icosahedral=icosahedral, subdir=subdir, + dependencies=analysis_dependencies) self.add_step(step, symlink=symlink) diff --git a/polaris/ocean/tasks/cosine_bell/analysis.py b/polaris/ocean/tasks/cosine_bell/analysis.py index 6e88a812b..f3f237f7e 100644 --- a/polaris/ocean/tasks/cosine_bell/analysis.py +++ b/polaris/ocean/tasks/cosine_bell/analysis.py @@ -5,6 +5,7 @@ import xarray as xr from polaris import Step +from polaris.ocean.resolution import resolution_to_subdir class Analysis(Step): @@ -13,14 +14,15 @@ class Analysis(Step): Attributes ---------- - resolutions : list of int + resolutions : list of float The resolutions of the meshes that have been run icosahedral : bool Whether to use icosahedral, as opposed to less regular, JIGSAW meshes """ - def __init__(self, component, resolutions, icosahedral, subdir): + def __init__(self, component, resolutions, icosahedral, subdir, + dependencies): """ Create the step @@ -29,7 +31,7 @@ def __init__(self, component, resolutions, icosahedral, subdir): component : polaris.Component The component the step belongs to - resolutions : list of int + resolutions : list of float The resolutions of the meshes that have been run icosahedral : bool @@ -38,22 +40,28 @@ def __init__(self, component, resolutions, icosahedral, subdir): subdir : str The subdirectory that the step resides in + + dependencies : dict of dict of polaris.Steps + The dependencies of this step """ super().__init__(component=component, name='analysis', subdir=subdir) self.resolutions = resolutions self.icosahedral = icosahedral for resolution in resolutions: - mesh_name = f'{resolution:g}km' + mesh_name = resolution_to_subdir(resolution) + base_mesh = dependencies['mesh'][resolution] + init = dependencies['init'][resolution] + forward = dependencies['forward'][resolution] self.add_input_file( filename=f'{mesh_name}_mesh.nc', - target=f'../base_mesh/{mesh_name}/base_mesh.nc') + work_dir_target=f'{base_mesh.path}/base_mesh.nc') self.add_input_file( filename=f'{mesh_name}_init.nc', - target=f'../init/{mesh_name}/initial_state.nc') + work_dir_target=f'{init.path}/initial_state.nc') self.add_input_file( filename=f'{mesh_name}_output.nc', - target=f'../forward/{mesh_name}/output.nc') + work_dir_target=f'{forward.path}/output.nc') self.add_output_file('convergence.png') @@ -65,8 +73,8 @@ def run(self): resolutions = self.resolutions xdata = list() ydata = list() - for res in resolutions: - mesh_name = f'{res:g}km' + for resolution in resolutions: + mesh_name = resolution_to_subdir(resolution) rmseValue, nCells = self.rmse(mesh_name) xdata.append(nCells) ydata.append(rmseValue) @@ -125,7 +133,8 @@ def rmse(self, mesh_name): lonCent = config.getfloat('cosine_bell', 'lon_center') radius = config.getfloat('cosine_bell', 'radius') psi0 = config.getfloat('cosine_bell', 'psi0') - pd = config.getfloat('cosine_bell', 'vel_pd') + convergence_eval_time = config.getfloat('spherical_convergence', + 'convergence_eval_time') ds_mesh = xr.open_dataset(f'{mesh_name}_mesh.nc') ds_init = xr.open_dataset(f'{mesh_name}_init.nc') @@ -135,7 +144,7 @@ def rmse(self, mesh_name): tt = str(ds.xtime[j].values) tt.rfind('_') DY = float(tt[10:12]) - 1 - if DY == pd: + if DY == convergence_eval_time: sliceTime = j break HR = float(tt[13:15]) @@ -144,7 +153,7 @@ def rmse(self, mesh_name): # find new location of blob center # center is based on equatorial velocity R = ds_mesh.sphere_radius - distTrav = 2.0 * 3.14159265 * R / (86400.0 * pd) * t + distTrav = 2.0 * 3.14159265 * R / (86400.0 * convergence_eval_time) * t # distance in radians is distRad = distTrav / R newLon = lonCent + distRad diff --git a/polaris/ocean/tasks/cosine_bell/cosine_bell.cfg b/polaris/ocean/tasks/cosine_bell/cosine_bell.cfg index 0231b5ca8..5909e8fa6 100644 --- a/polaris/ocean/tasks/cosine_bell/cosine_bell.cfg +++ b/polaris/ocean/tasks/cosine_bell/cosine_bell.cfg @@ -19,12 +19,33 @@ partial_cell_type = None # The minimum fraction of a layer for partial cells min_pc_fraction = 0.1 + +# config options for spherical convergence tests +[spherical_convergence] + +# Evaluation time for convergence analysis (in days) +convergence_eval_time = ${cosine_bell:vel_pd} + + +# config options for spherical convergence tests +[spherical_convergence_forward] + +# time integrator: {'split_explicit', 'RK4'} +time_integrator = RK4 + +# RK4 time step per resolution (s/km), since dt is proportional to resolution +rk4_dt_per_km = 3.0 + +# Run duration in days +run_duration = ${cosine_bell:vel_pd} + +# Output interval in days +output_interval = ${cosine_bell:vel_pd} + + # options for cosine bell convergence test case [cosine_bell] -# time step per resolution (s/km), since dt is proportional to resolution -dt_per_km = 30 - # the constant temperature of the domain temperature = 15.0 diff --git a/polaris/ocean/tasks/cosine_bell/forward.py b/polaris/ocean/tasks/cosine_bell/forward.py index c93bcc19b..5212332d0 100644 --- a/polaris/ocean/tasks/cosine_bell/forward.py +++ b/polaris/ocean/tasks/cosine_bell/forward.py @@ -1,23 +1,13 @@ -import time +from polaris.ocean.convergence.spherical import SphericalConvergenceForward -from polaris.ocean.model import OceanModelStep - -class Forward(OceanModelStep): +class Forward(SphericalConvergenceForward): """ A step for performing forward ocean component runs as part of the cosine bell test case - - Attributes - ---------- - resolution : int - The resolution of the (uniform) mesh in km - - mesh_name : str - The name of the mesh """ - def __init__(self, component, name, subdir, resolution, mesh_name): + def __init__(self, component, name, subdir, resolution, base_mesh, init): """ Create a new step @@ -32,75 +22,20 @@ def __init__(self, component, name, subdir, resolution, mesh_name): subdir : str The subdirectory for the step - resolution : int + resolution : float The resolution of the (uniform) mesh in km - mesh_name : str - The name of the mesh - """ - super().__init__(component=component, name=name, subdir=subdir, - openmp_threads=1) - - self.resolution = resolution - self.mesh_name = mesh_name + base_mesh : polaris.Step + The base mesh step - # make sure output is double precision - self.add_yaml_file('polaris.ocean.config', 'output.yaml') - self.add_yaml_file( - 'polaris.ocean.tasks.cosine_bell', - 'forward.yaml') - - self.add_input_file( - filename='init.nc', - target=f'../../init/{mesh_name}/initial_state.nc') - self.add_input_file( - filename='graph.info', - target=f'../../../base_mesh/{mesh_name}/graph.info') - - self.add_output_file(filename='output.nc', - validate_vars=['normalVelocity', 'tracer1']) - - def compute_cell_count(self): + init : polaris.Step + The init step """ - Compute the approximate number of cells in the mesh, used to constrain - resources - - Returns - ------- - cell_count : int or None - The approximate number of cells in the mesh - """ - # use a heuristic based on QU30 (65275 cells) and QU240 (10383 cells) - cell_count = 6e8 / self.resolution**2 - return cell_count - - def dynamic_model_config(self, at_setup): - """ - Set the model time step from config options at setup and runtime - - Parameters - ---------- - at_setup : bool - Whether this method is being run during setup of the step, as - opposed to at runtime - """ - super().dynamic_model_config(at_setup=at_setup) - - config = self.config - - vert_levels = config.getfloat('vertical_grid', 'vert_levels') - if not at_setup and vert_levels == 1: - self.add_yaml_file('polaris.ocean.config', 'single_layer.yaml') - self.add_yaml_file( - 'polaris.ocean.tasks.cosine_bell', - 'forward.yaml') - - # dt is proportional to resolution: default 30 seconds per km - dt_per_km = config.getfloat('cosine_bell', 'dt_per_km') - - dt = dt_per_km * self.resolution - # https://stackoverflow.com/a/1384565/7728169 - dt_str = time.strftime('%H:%M:%S', time.gmtime(dt)) - - options = dict(config_dt=dt_str) - self.add_model_config_options(options) + package = 'polaris.ocean.tasks.cosine_bell' + validate_vars = ['normalVelocity', 'tracer1'] + super().__init__(component=component, name=name, subdir=subdir, + resolution=resolution, base_mesh=base_mesh, + init=init, package=package, + yaml_filename='forward.yaml', + output_filename='output.nc', + validate_vars=validate_vars) diff --git a/polaris/ocean/tasks/cosine_bell/forward.yaml b/polaris/ocean/tasks/cosine_bell/forward.yaml index f72c034c7..f77e92227 100644 --- a/polaris/ocean/tasks/cosine_bell/forward.yaml +++ b/polaris/ocean/tasks/cosine_bell/forward.yaml @@ -2,11 +2,12 @@ omega: run_modes: config_ocean_run_mode: forward time_management: - config_run_duration: 0024_00:00:00 + config_run_duration: {{ run_duration }} decomposition: config_block_decomp_file_prefix: graph.info.part. time_integration: - config_time_integrator: RK4 + config_dt: {{ dt }} + config_time_integrator: {{ time_integrator }} debug: config_disable_thick_all_tend: true config_disable_thick_hadv: true @@ -50,7 +51,7 @@ omega: output: type: output filename_template: output.nc - output_interval: 0024_00:00:00 + output_interval: {{ output_interval }} clobber_mode: truncate reference_time: 0001-01-01_00:00:00 contents: diff --git a/polaris/ocean/tasks/cosine_bell/init.py b/polaris/ocean/tasks/cosine_bell/init.py index 306b3608a..943b6596d 100644 --- a/polaris/ocean/tasks/cosine_bell/init.py +++ b/polaris/ocean/tasks/cosine_bell/init.py @@ -10,7 +10,7 @@ class Init(Step): """ A step for an initial condition for for the cosine bell test case """ - def __init__(self, component, name, subdir, mesh_name): + def __init__(self, component, name, subdir, base_mesh): """ Create the step @@ -25,18 +25,18 @@ def __init__(self, component, name, subdir, mesh_name): subdir : str The subdirectory for the step - mesh_name : str - The name of the mesh + base_mesh : polaris.Step + The base mesh step """ super().__init__(component=component, name=name, subdir=subdir) self.add_input_file( filename='mesh.nc', - target=f'../../../base_mesh/{mesh_name}/base_mesh.nc') + work_dir_target=f'{base_mesh.path}/base_mesh.nc') self.add_input_file( filename='graph.info', - target=f'../../../base_mesh/{mesh_name}/graph.info') + work_dir_target=f'{base_mesh.path}/graph.info') self.add_output_file(filename='initial_state.nc') diff --git a/polaris/ocean/tasks/cosine_bell/viz.py b/polaris/ocean/tasks/cosine_bell/viz.py index 4f81aadef..503bebcf2 100644 --- a/polaris/ocean/tasks/cosine_bell/viz.py +++ b/polaris/ocean/tasks/cosine_bell/viz.py @@ -15,7 +15,7 @@ class VizMap(MappingFileStep): mesh_name : str The name of the mesh """ - def __init__(self, component, name, subdir, mesh_name): + def __init__(self, component, name, subdir, base_mesh, mesh_name): """ Create the step @@ -30,6 +30,9 @@ def __init__(self, component, name, subdir, mesh_name): subdir : str The subdirectory for the step + base_mesh : polaris.Step + The base mesh step + mesh_name : str The name of the mesh """ @@ -38,7 +41,7 @@ def __init__(self, component, name, subdir, mesh_name): self.mesh_name = mesh_name self.add_input_file( filename='mesh.nc', - target=f'../../../../base_mesh/{mesh_name}/base_mesh.nc') + work_dir_target=f'{base_mesh.path}/base_mesh.nc') def runtime_setup(self): """ @@ -65,7 +68,8 @@ class Viz(Step): mesh_name : str The name of the mesh """ - def __init__(self, component, name, subdir, viz_map, mesh_name): + def __init__(self, component, name, subdir, base_mesh, init, forward, + viz_map, mesh_name): """ Create the step @@ -80,6 +84,15 @@ def __init__(self, component, name, subdir, viz_map, mesh_name): subdir : str The subdirectory in the test case's work directory for the step + base_mesh : polaris.Step + The base mesh step + + init : polaris.Step + The init step + + forward : polaris.Step + The init step + viz_map : polaris.ocean.tasks.cosine_bell.viz.VizMap The step for creating a mapping files, also used to remap data from the MPAS mesh to a lon-lat grid @@ -90,13 +103,13 @@ def __init__(self, component, name, subdir, viz_map, mesh_name): super().__init__(component=component, name=name, subdir=subdir) self.add_input_file( filename='mesh.nc', - target=f'../../../../base_mesh/{mesh_name}/base_mesh.nc') + work_dir_target=f'{base_mesh.path}/base_mesh.nc') self.add_input_file( filename='initial_state.nc', - target=f'../../../init/{mesh_name}/initial_state.nc') + work_dir_target=f'{init.path}/initial_state.nc') self.add_input_file( filename='output.nc', - target=f'../../../forward/{mesh_name}/output.nc') + work_dir_target=f'{forward.path}/output.nc') self.add_dependency(viz_map, name='viz_map') self.mesh_name = mesh_name self.add_output_file('init.png') @@ -108,7 +121,8 @@ def run(self): """ config = self.config mesh_name = self.mesh_name - period = config.getfloat('cosine_bell', 'vel_pd') + run_duration = config.getfloat('spherical_convergence_forward', + 'run_duration') viz_map = self.dependencies['viz_map'] @@ -134,5 +148,5 @@ def run(self): ds_out.tracer1.values, out_filename='final.png', config=config, colormap_section='cosine_bell_viz', - title=f'{mesh_name} tracer after {period} days', + title=f'{mesh_name} tracer after {run_duration:g} days', plot_land=False) From f639d4262788eb582f69ed1b9efe8b2f86c55fd3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 21 Sep 2023 09:43:15 +0200 Subject: [PATCH 4/4] Update the docs --- docs/developers_guide/framework/config.md | 16 +- docs/developers_guide/ocean/api.md | 15 +- docs/developers_guide/ocean/framework.md | 223 ++++++++++++++++++ .../ocean/tasks/cosine_bell.md | 14 +- docs/developers_guide/organization/tasks.md | 34 +-- docs/users_guide/ocean/tasks/cosine_bell.md | 75 ++++-- 6 files changed, 329 insertions(+), 48 deletions(-) diff --git a/docs/developers_guide/framework/config.md b/docs/developers_guide/framework/config.md index e54518d35..7cc386126 100644 --- a/docs/developers_guide/framework/config.md +++ b/docs/developers_guide/framework/config.md @@ -33,18 +33,16 @@ the file is in the path `polaris/ocean/tasks/baroclinic_channel` that the config file should always exist, so we would like the code to raise an exception (`exception=True`) if the file is not found. This is the default behavior. In some cases, you would like the code to add the config -options if the config file exists and do nothing if it does not. This can -be useful if a common configure function is being used for all test -cases in a configuration, as in this example from -{py:func}`setup.setup_task()`: +options if the config file exists and do nothing if it does not. In this +example from {py:func}`polaris.setup.setup_task()`, there may not be a config +file for the particular machine we're on, and that's fine: ```python -# add the config options for the task (if defined) -config.add_from_package(task.__module__, - f'{task.name}.cfg', exception=False) +if machine is not None: + config.add_from_package('mache.machines', f'{machine}.cfg', + exception=False) ``` - -If a task doesn't have any config options, nothing will happen. +If there isn't a config file for this machine, nothing will happen. The `MpasConfigParser` class also includes methods for adding a user config file and other config files by file name, but these are largely intended diff --git a/docs/developers_guide/ocean/api.md b/docs/developers_guide/ocean/api.md index fcde46b33..812efda9b 100644 --- a/docs/developers_guide/ocean/api.md +++ b/docs/developers_guide/ocean/api.md @@ -169,7 +169,18 @@ ## Ocean Framework -### OceanModelStep +### Spherical Convergence Tests + +```{eval-rst} +.. currentmodule:: polaris.ocean.convergence.spherical + +.. autosummary:: + :toctree: generated/ + + SphericalConvergenceForward +``` + +### Ocean Model ```{eval-rst} .. currentmodule:: polaris.ocean.model @@ -182,6 +193,8 @@ OceanModelStep.constrain_resources OceanModelStep.compute_cell_count OceanModelStep.map_yaml_to_namelist + + get_time_interval_string ``` ### Spherical Base Mesh Step diff --git a/docs/developers_guide/ocean/framework.md b/docs/developers_guide/ocean/framework.md index 3598a4905..14ee03786 100644 --- a/docs/developers_guide/ocean/framework.md +++ b/docs/developers_guide/ocean/framework.md @@ -60,6 +60,57 @@ The config options `goal_cells_per_core` and `max_cells_per_core` in the the planar mesh. By default, the number of MPI tasks tries to apportion 200 cells to each core, but it will allow as many as 2000. +### Setting time intervals in model config options + +It is often useful to be able to convert a `float` time interval in days or +seconds to a model config option in the form `DDDD_HH:MM:SS.S`. The +{py:func}`polaris.ocean.model.get_time_interval_string()` function will do this +for you. For example, if you have `resolution` in km and a config `section` +with options `dt_per_km` (in s/km) and `run_duration` (in days), you can use +the function to get appropriate strings for filling in a template model config +file: +```python +from polaris.ocean.model import get_time_interval_string + + +dt_per_km = section.getfloat('dt_per_km') +dt_str = get_time_interval_string(seconds=dt_per_km * resolution) + +run_duration = section.getfloat('run_duration') +run_duration_str = get_time_interval_string(days=run_duration) + +output_interval = section.getfloat('output_interval') +output_interval_str = get_time_interval_string(days=output_interval) + +replacements = dict( + dt=dt_str, + run_duration=run_duration_str, + output_interval=output_interval_str +) + +self.add_yaml_file(package, yaml_filename, + template_replacements=replacements) +``` +where the YAML file might include: +``` +omega: + time_management: + config_run_duration: {{ run_duration }} + time_integration: + config_dt: {{ dt }} + streams: + output: + type: output + filename_template: output.nc + output_interval: {{ output_interval }} + clobber_mode: truncate + reference_time: 0001-01-01_00:00:00 + contents: + - xtime + - normalVelocity + - layerThickness +``` + (dev-ocean-framework-config)= ## Model config options and streams @@ -85,6 +136,178 @@ The function {py:func}`polaris.ocean.mesh.spherical.add_spherical_base_mesh_step returns a step for for a spherical `qu` or `icos` mesh of a given resolution (in km). The step can be shared between tasks. +(dev-ocean-spherical-convergence)= + +## Spherical Convergence Tests + +Several tests that are in Polaris or which we plan to add are convergence +tests on {ref}`dev-ocean-spherical-meshes`. The ocean framework includes +shared config options and a base class for forward steps that are expected +to be useful across these tests. + +The shared config options are: +```cfg +# config options for spherical convergence tests +[spherical_convergence] + +# a list of icosahedral mesh resolutions (km) to test +icos_resolutions = 60, 120, 240, 480 + +# a list of quasi-uniform mesh resolutions (km) to test +qu_resolutions = 60, 90, 120, 150, 180, 210, 240 + +# Evaluation time for convergence analysis (in days) +convergence_eval_time = 1.0 + + +# config options for spherical convergence forward steps +[spherical_convergence_forward] + +# time integrator: {'split_explicit', 'RK4'} +time_integrator = RK4 + +# RK4 time step per resolution (s/km), since dt is proportional to resolution +rk4_dt_per_km = 3.0 + +# split time step per resolution (s/km), since dt is proportional to resolution +split_dt_per_km = 30.0 + +# the barotropic time step (s/km) for simulations using split time stepping, +# since btr_dt is proportional to resolution +btr_dt_per_km = 1.5 + +# Run duration in days +run_duration = ${spherical_convergence:convergence_eval_time} + +# Output interval in days +output_interval = ${run_duration} +``` +The first 2 are the default resolutions for icosahedral and quasi-uniform +base meshes, respectively. The `time_integrator` will typically be overridden +by the specific convergence task's config options, and indicates which time +integrator to use for the forward run. Depending on the time integrator, +either `rk4_dt_per_km` or `split_dt_per_km` will be used to determine an +appropriate time step for each mesh resolution (proportional to the cell size). +For split time integrators, `btr_dt_per_km` will be used to compute the +barotropic time step in a similar way. The `run_duration` and +`output_interval` are typically the same, and they are given in days. + +Each convergence test can override these defaults with its own defaults by +defining them in its own config file. Convergence tests should bring in this +config file in their `configure()` methods, then add its own config options +after that to make sure they take precedence, e.g.: + +```python +from polaris import Task +class CosineBell(Task): + def configure(self): + super().configure() + config = self.config + config.add_from_package('polaris.mesh', 'mesh.cfg') + config.add_from_package('polaris.ocean.convergence.spherical', + 'spherical.cfg') + config.add_from_package('polaris.ocean.tasks.cosine_bell', + 'cosine_bell.cfg') +``` + +In addition, the {py:class}`polaris.ocean.convergence.spherical.SphericalConvergenceForward` +step can serve as a parent class for forward steps in convergence tests. This +parent class takes care of setting the time step based on the `dt_per_km` +config option and computes the approximate number of cells in the mesh, used +for determining the computational resources required, using a heuristic +appropriate for approximately uniform spherical meshes. A convergence test's +`Forward` step should descend from this class like in this example: + +```python +from polaris.ocean.convergence.spherical import SphericalConvergenceForward + + +class Forward(SphericalConvergenceForward): + """ + A step for performing forward ocean component runs as part of the cosine + bell test case + """ + + def __init__(self, component, name, subdir, resolution, base_mesh, init): + """ + Create a new step + + Parameters + ---------- + component : polaris.Component + The component the step belongs to + + name : str + The name of the step + + subdir : str + The subdirectory for the step + + resolution : float + The resolution of the (uniform) mesh in km + + base_mesh : polaris.Step + The base mesh step + + init : polaris.Step + The init step + """ + package = 'polaris.ocean.tasks.cosine_bell' + validate_vars = ['normalVelocity', 'tracer1'] + super().__init__(component=component, name=name, subdir=subdir, + resolution=resolution, base_mesh=base_mesh, + init=init, package=package, + yaml_filename='forward.yaml', + output_filename='output.nc', + validate_vars=validate_vars) +``` +Each convergence test must define a YAML file with model config options, called +`forward.yaml` by default. The `package` parameter is the location of this +file within the Polaris code (using python package syntax). Although it is +not used here, the `options` parameter can be used to pass model config options +as a python dictionary so that they are added to with +{py:meth}`polaris.ModelStep.add_model_config_options()`. The +`output_filename` is an output file that will have fields to validate and +analyze. The `validate_vars` are a list of variables to compare against a +baseline (if one is provided), and can be `None` if baseline validation should +not be performed. + +The `base_mesh` step should be created with the function described in +{ref}`dev-ocean-spherical-meshes`, and the `init` step should produce a file +`initial_state.nc` that will be the initial condition for the forward run. + +The `forward.yaml` file should be a YAML file with Jinja templating for the +time integrator, time step, run duration and output interval, e.g.: +``` +omega: + time_management: + config_run_duration: {{ run_duration }} + time_integration: + config_dt: {{ dt }} + config_time_integrator: {{ time_integrator }} + split_explicit_ts: + config_btr_dt: {{ btr_dt }} + streams: + mesh: + filename_template: init.nc + input: + filename_template: init.nc + restart: {} + output: + type: output + filename_template: output.nc + output_interval: {{ output_interval }} + clobber_mode: truncate + reference_time: 0001-01-01_00:00:00 + contents: + - xtime + - normalVelocity + - layerThickness +``` +`SphericalConvergenceForward` takes care of filling in the template based +on the associated config options (first at setup and again at runtime in case +the config options have changed). + (dev-ocean-framework-vertical)= ## Vertical coordinate diff --git a/docs/developers_guide/ocean/tasks/cosine_bell.md b/docs/developers_guide/ocean/tasks/cosine_bell.md index fe88582d7..180012744 100644 --- a/docs/developers_guide/ocean/tasks/cosine_bell.md +++ b/docs/developers_guide/ocean/tasks/cosine_bell.md @@ -16,7 +16,9 @@ The config options for the `cosine_bell` tests are described in Additionally, the test uses a `forward.yaml` file with a few common model config options related to drag and default horizontal and vertical momentum and tracer diffusion, as well as defining `mesh`, `input`, -`restart`, and `output` streams. +`restart`, and `output` streams. This file has Jinja templating that is +used to update model config options based on Polaris config options, see +{ref}`dev-ocean-spherical-convergence`. ### base_mesh @@ -32,10 +34,12 @@ tracer distributed in a cosine-bell shape. ### forward The class {py:class}`polaris.ocean.tasks.cosine_bell.forward.Forward` -defines a step for running MPAS-Ocean from an initial condition produced in -an `init` step. The time step is determined from the resolution -based on the `dt_per_km` config option. Other namelist options are taken -from the task's `forward.yaml`. +descends from {py:class}`polaris.ocean.convergence.spherical.SphericalConvergenceForward`, +and defines a step for running MPAS-Ocean from an initial condition produced in +an `init` step. See {ref}`dev-ocean-spherical-convergence` for some relevant +discussion of the parent class. The time step is determined from the resolution +based on the `dt_per_km` config option in the `[spherical_convergences]` +section. Other model config options are taken from `forward.yaml`. ### analysis diff --git a/docs/developers_guide/organization/tasks.md b/docs/developers_guide/organization/tasks.md index 61cf9f368..c69775f20 100644 --- a/docs/developers_guide/organization/tasks.md +++ b/docs/developers_guide/organization/tasks.md @@ -459,10 +459,21 @@ config files within the task or its shared framework. The `self.config` attribute that is modified in this function will be written to a config file for the task (see {ref}`config-files`). -If you override this method in a task, you should assume that the -`.cfg` file in its package has already been added to the -config options prior to calling `configure()`. This happens automatically -before running the task. +If you define a `.cfg` file, you will want to override this method +to add those config options, e.g.: + +```python +from polaris import Task + +class InertialGravityWave(Task): + def configure(self): + """ + Add the config file common to inertial gravity wave tests + """ + self.config.add_from_package( + 'polaris.ocean.tasks.inertial_gravity_wave', + 'inertial_gravity_wave.cfg') +``` Since many tasks may need similar behavior in their `configure()` methods, it is sometimes useful to define a parent class that overrides the @@ -472,24 +483,19 @@ the `configure()` method with their own additional changes. A `configure()` method can also be used to perform other operations at the task level when a task is being set up. An example of this would be -creating a symlink to a README file that is shared across the whole task, -as in {py:meth}`polaris.ocean.tasks.global_ocean.files_for_e3sm.FilesForE3SM.configure()`: +creating a symlink to a README file that is shared across the whole task: ```python -from importlib.resources import path - -from polaris.ocean.tasks.global_ocean.configure import configure_global_ocean -from polaris.io import symlink +from polaris.io import imp_res, symlink def configure(self): """ Modify the configuration options for this task """ - configure_global_ocean(task=self, mesh=self.mesh, init=self.init) - with path('polaris.ocean.tasks.global_ocean.files_for_e3sm', - 'README') as target: - symlink(str(target), '{}/README'.format(self.work_dir)) + package = 'compass.ocean.tests.global_ocean.files_for_e3sm' + target = imp_res.files(package).joinpath('README') + symlink(str(target), f'{self.work_dir}/README') ``` The `configure()` method is not the right place for adding or modifying steps diff --git a/docs/users_guide/ocean/tasks/cosine_bell.md b/docs/users_guide/ocean/tasks/cosine_bell.md index 7d44357dd..fd1c44b11 100644 --- a/docs/users_guide/ocean/tasks/cosine_bell.md +++ b/docs/users_guide/ocean/tasks/cosine_bell.md @@ -41,24 +41,33 @@ The default resolutions used in the task depends on the mesh type. For the `icos` mesh type, the defaults are: ```cfg -resolutions = 60, 120, 240, 480 +# config options for spherical convergence tests +[spherical_convergence] + +# a list of icosahedral mesh resolutions (km) to test +icos_resolutions = 60, 120, 240, 480 ``` for the `qu` mesh type, they are: ```cfg -resolutions = 60, 90, 120, 150, 180, 210, 240 +# config options for spherical convergence tests +[spherical_convergence] + +# a list of quasi-uniform mesh resolutions (km) to test +qu_resolutions = 60, 90, 120, 150, 180, 210, 240 ``` To alter the resolutions used in this task, you will need to create your own -config file (or add a `cosine_bell` section to a config file if you're -already using one). The resolutions are a comma-separated list of the +config file (or add a `spherical_convergence` section to a config file if +you're already using one). The resolutions are a comma-separated list of the resolution of the mesh in km. If you specify a different list before setting up `cosine_bell`, steps will be generated with the requested -resolutions. (If you alter `resolutions` in the task's config file in -the work directory, nothing will happen.) For `icos` meshes, make sure you -use a resolution close to those listed in {ref}`dev-spherical-meshes`. Each -resolution will be rounded to the nearest allowed icosahedral resolution. +resolutions. (If you alter `icos_resolutions` or `qu_resolutions`) in the +task's config file in the work directory, nothing will happen.) For `icos` +meshes, make sure you use a resolution close to those listed in +{ref}`dev-spherical-meshes`. Each resolution will be rounded to the nearest +allowed icosahedral resolution. The `base_mesh` steps are shared with other tasks so they are not housed in the `cosine_bell` work directory. Instead, they are in work directories like: @@ -146,11 +155,45 @@ field remains at the initial velocity $u_0$. ## time step and run duration -The time step for forward integration is determined by multiplying the -resolution by `dt_per_km`, so that coarser meshes have longer time steps. -You can alter this before setup (in a user config file) or before running the -task (in the config file in the work directory). The run duration is 24 -days. +This task uses the Runge-Kutta 4th-order (RK4) time integrator. The time step +for forward integration is determined by multiplying the resolution by a config +option, `rk4_dt_per_km`, so that coarser meshes have longer time steps. You can +alter this before setup (in a user config file) or before running the task (in +the config file in the work directory). + +```cfg +# config options for spherical convergence tests +[spherical_convergence_forward] + +# time integrator: {'split_explicit', 'RK4'} +time_integrator = RK4 + +# RK4 time step per resolution (s/km), since dt is proportional to resolution +rk4_dt_per_km = 3.0 +``` + +The convergence_eval_time, run duration and output interval are the period for +advection to make a full rotation around the globe, 24 days: + +```cfg +# config options for spherical convergence tests +[spherical_convergence] + +# Evaluation time for convergence analysis (in days) +convergence_eval_time = ${cosine_bell:vel_pd} + +# config options for spherical convergence tests +[spherical_convergence_forward] + +# Run duration in days +run_duration = ${cosine_bell:vel_pd} + +# Output interval in days +output_interval = ${cosine_bell:vel_pd} +``` + +Her, `${cosine_bell:vel_pd}` means that the same value is used as in the +option `vel_pd` in section `[cosine_bell]`, see below. ## config options @@ -160,9 +203,6 @@ The `cosine_bell` config options include: # options for cosine bell convergence test case [cosine_bell] -# time step per resolution (s/km), since dt is proportional to resolution -dt_per_km = 30 - # the constant temperature of the domain temperature = 15.0 @@ -221,9 +261,6 @@ norm_args = {'vmin': 0., 'vmax': 1.} # colorbar_ticks = np.linspace(0., 1., 9) ``` -The `dt_per_km` option `[cosine_bell]` is used to control the time step, as -discussed above in more detail. - The 7 options from `temperature` to `vel_pd` are used to control properties of the cosine bell and the rest of the sphere, as well as the advection.