diff --git a/config/nowcast.yaml b/config/nowcast.yaml index 49b596fa..46832f9e 100644 --- a/config/nowcast.yaml +++ b/config/nowcast.yaml @@ -749,10 +749,6 @@ run: wave forecasts: # Compute host to run wave forecast on host: arbutus.cloud-nowcast - # NEMO run to run wwatch3 forecast run after - # User 'after forecast' during storm surge season and 'after nowcast-green` - # the rest of the year - run when: after nowcast-green # Directory on compute host where files (e.g. ww3_*.inp, mod_def.ww3) and # directories (e,g. wind/ current/) necessary to prepare the wwatch3 runs # are stored @@ -768,6 +764,10 @@ wave forecasts: # Template for NEMO hourly results file name # **Must be quoted to project {} characters** NEMO file template: 'SalishSea_1h_{s_yyyymmdd}_{e_yyyymmdd}_grid_{grid}.nc' + # Location on the compute host of the file that contains IP addresses + # and MPI slots specifications. + # Only required for runs on cloud hosts. + mpi hosts file: ${HOME}/mpi_hosts.wwatch3 # Path to the wwatch3 executables directory wwatch3 exe path: /nemoShare/MEOPAR/nowcast-sys/wwatch3-5.16/exe # Path to the salishsea command processor executable to use in the run script diff --git a/nowcast/next_workers.py b/nowcast/next_workers.py index 16c04b26..6b3975a2 100644 --- a/nowcast/next_workers.py +++ b/nowcast/next_workers.py @@ -789,7 +789,6 @@ def after_watch_NEMO(msg, config, checklist): race_condition_workers = {} if msg.type.startswith("success"): run_type = msg.type.split()[1] - wave_forecast_after = config["wave forecasts"]["run when"].split("after ")[1] if run_type == "nowcast": next_workers[msg.type].extend( [ @@ -811,33 +810,32 @@ def after_watch_NEMO(msg, config, checklist): ] ) if run_type == "forecast": - if wave_forecast_after == "forecast": - host_name = config["wave forecasts"]["host"] - next_workers[msg.type].extend( - [ - NextWorker( - "nowcast.workers.make_ww3_wind_file", - args=[ - host_name, - "forecast", - "--run-date", - msg.payload[run_type]["run date"], - ], - host=host_name, - ), - NextWorker( - "nowcast.workers.make_ww3_current_file", - args=[ - host_name, - "forecast", - "--run-date", - msg.payload[run_type]["run date"], - ], - host=host_name, - ), - ] - ) - race_condition_workers = {"make_ww3_wind_file", "make_ww3_current_file"} + host_name = config["wave forecasts"]["host"] + next_workers[msg.type].extend( + [ + NextWorker( + "nowcast.workers.make_ww3_wind_file", + args=[ + host_name, + "forecast", + "--run-date", + msg.payload[run_type]["run date"], + ], + host=host_name, + ), + NextWorker( + "nowcast.workers.make_ww3_current_file", + args=[ + host_name, + "forecast", + "--run-date", + msg.payload[run_type]["run date"], + ], + host=host_name, + ), + ] + ) + race_condition_workers = {"make_ww3_wind_file", "make_ww3_current_file"} for host in config["run"]["enabled hosts"]: if not config["run"]["enabled hosts"][host]["shared storage"]: next_workers[msg.type].append( @@ -852,33 +850,6 @@ def after_watch_NEMO(msg, config, checklist): ) ) if run_type == "nowcast-green": - if wave_forecast_after == "nowcast-green": - host_name = config["wave forecasts"]["host"] - next_workers[msg.type].extend( - [ - NextWorker( - "nowcast.workers.make_ww3_wind_file", - args=[ - host_name, - "forecast", - "--run-date", - msg.payload[run_type]["run date"], - ], - host=host_name, - ), - NextWorker( - "nowcast.workers.make_ww3_current_file", - args=[ - host_name, - "forecast", - "--run-date", - msg.payload[run_type]["run date"], - ], - host=host_name, - ), - ] - ) - race_condition_workers = {"make_ww3_wind_file", "make_ww3_current_file"} for host in config["run"]["enabled hosts"]: run_types = config["run"]["enabled hosts"][host]["run types"] if "nowcast-dev" in run_types: diff --git a/nowcast/workers/make_ww3_current_file.py b/nowcast/workers/make_ww3_current_file.py index d6450f6c..05b0f909 100644 --- a/nowcast/workers/make_ww3_current_file.py +++ b/nowcast/workers/make_ww3_current_file.py @@ -97,20 +97,13 @@ def make_ww3_current_file(parsed_args, config, *args): mesh_mask = os.fspath(grid_dir / config["run types"]["nowcast"]["mesh mask"]) nemo_dir = Path(host_config["run types"]["nowcast"]["results"]).parent nemo_file_tmpl = config["wave forecasts"]["NEMO file template"] - wave_forecast_after = config["wave forecasts"]["run when"].split("after ")[1] dest_dir = Path(config["wave forecasts"]["run prep dir"], "current") filepath_tmpl = config["wave forecasts"]["current file template"] nc_filepath = dest_dir / filepath_tmpl.format(yyyymmdd=run_date.format("YYYYMMDD")) if run_type in {"nowcast", "forecast"}: - datasets = _calc_nowcast_datasets( - run_date, nemo_dir, nemo_file_tmpl, wave_forecast_after - ) + datasets = _calc_nowcast_datasets(run_date, nemo_dir, nemo_file_tmpl) if run_type == "forecast": - datasets.update( - _calc_forecast_datasets( - run_date, nemo_dir, nemo_file_tmpl, wave_forecast_after - ) - ) + datasets.update(_calc_forecast_datasets(run_date, nemo_dir, nemo_file_tmpl)) if run_type == "forecast2": datasets = _calc_forecast2_datasets( run_date, nemo_dir, nemo_file_tmpl, dest_dir @@ -119,12 +112,51 @@ def make_ww3_current_file(parsed_args, config, *args): lats = grid.nav_lat[1:, 1:] lons = grid.nav_lon[1:, 1:] + 360 logger.debug(f"lats and lons from: {mesh_mask}") - with xarray.open_mfdataset(datasets["u"]) as u_nemo: + drop_vars = { + "area", + "bounds_lon", + "bounds_lat", + "bounds_nav_lon", + "bounds_nav_lat", + "depthu_bounds", + "depthv_bounds", + "time_centered_bounds", + "time_counter_bounds", + } + chunks = { + "u": { + "time_counter": 3, + "depthu": 40, + "y": 898, + "x": 398, + }, + "v": { + "time_counter": 3, + "depthv": 40, + "y": 898, + "x": 398, + }, + } + with xarray.open_mfdataset( + datasets["u"], + chunks=chunks["u"], + compat="override", + coords="minimal", + data_vars="minimal", + drop_variables=drop_vars, + ) as u_nemo: logger.debug(f'u velocities from {datasets["u"]}') - with xarray.open_mfdataset(datasets["v"]) as v_nemo: + with xarray.open_mfdataset( + datasets["v"], + chunks=chunks["v"], + compat="override", + coords="minimal", + data_vars="minimal", + drop_variables=drop_vars, + ) as v_nemo: logger.debug(f'v velocities from {datasets["v"]}') u_unstaggered, v_unstaggered = viz_tools.unstagger( - u_nemo.vozocrtx[:, 0, ...], v_nemo.vomecrty[:, 0, ...] + u_nemo.vozocrtx.isel(depthu=0), v_nemo.vomecrty.isel(depthv=0) ) del u_unstaggered.coords["time_centered"] del u_unstaggered.coords["depthu"] @@ -146,15 +178,14 @@ def make_ww3_current_file(parsed_args, config, *args): return checklist -def _calc_nowcast_datasets(run_date, nemo_dir, nemo_file_tmpl, wave_forecast_after): +def _calc_nowcast_datasets(run_date, nemo_dir, nemo_file_tmpl): datasets = {"u": [], "v": []} - nemo_results = "nowcast" if wave_forecast_after == "forecast" else "nowcast-green" dmy = run_date.format("DDMMMYY").lower() s_yyyymmdd = e_yyyymmdd = run_date.format("YYYYMMDD") for grid in datasets: nowcast_file = ( nemo_dir - / Path(nemo_results, dmy) + / Path("nowcast", dmy) / nemo_file_tmpl.format( s_yyyymmdd=s_yyyymmdd, e_yyyymmdd=e_yyyymmdd, grid=grid.upper() ) @@ -164,15 +195,14 @@ def _calc_nowcast_datasets(run_date, nemo_dir, nemo_file_tmpl, wave_forecast_aft return datasets -def _calc_forecast_datasets(run_date, nemo_dir, nemo_file_tmpl, wave_forecast_after): +def _calc_forecast_datasets(run_date, nemo_dir, nemo_file_tmpl): datasets = {"u": [], "v": []} - nemo_results = "nowcast" if wave_forecast_after == "forecast" else "nowcast-green" dmy = run_date.format("DDMMMYY").lower() s_yyyymmdd = e_yyyymmdd = run_date.format("YYYYMMDD") for grid in datasets: nowcast_file = ( nemo_dir - / Path(nemo_results, dmy) + / Path("nowcast", dmy) / nemo_file_tmpl.format( s_yyyymmdd=s_yyyymmdd, e_yyyymmdd=e_yyyymmdd, grid=grid.upper() ) diff --git a/nowcast/workers/make_ww3_wind_file.py b/nowcast/workers/make_ww3_wind_file.py index 4937a67a..25345f64 100644 --- a/nowcast/workers/make_ww3_wind_file.py +++ b/nowcast/workers/make_ww3_wind_file.py @@ -114,7 +114,30 @@ def make_ww3_wind_file(parsed_args, config, *args): lats = lats_lons.nav_lat lons = lats_lons.nav_lon logger.debug(f"lats and lons from: {datasets[0]}") - with xarray.open_mfdataset(datasets) as hrdps: + drop_vars = { + "LHTFL_surface", + "PRATE_surface", + "RH_2maboveground", + "atmpres", + "precip", + "qair", + "solar", + "tair", + "therm_rad", + } + chunks = { + "time_counter": 1, + "y": 230, + "x": 190, + } + with xarray.open_mfdataset( + datasets, + chunks=chunks, + compat="override", + coords="minimal", + data_vars="minimal", + drop_variables=drop_vars, + ) as hrdps: ds = _create_dataset( hrdps.time_counter, lats, lons, hrdps.u_wind, hrdps.v_wind, datasets ) diff --git a/nowcast/workers/run_ww3.py b/nowcast/workers/run_ww3.py index e62e465a..e807686f 100644 --- a/nowcast/workers/run_ww3.py +++ b/nowcast/workers/run_ww3.py @@ -463,13 +463,14 @@ def _definitions(run_date, run_type, run_dir_path, results_path, config): """ ddmmmyy = run_date.format("DDMMMYY").lower() wwatch3_exe_path = config["wave forecasts"]["wwatch3 exe path"] + mpi_hosts_file = config["wave forecasts"]["mpi hosts file"] salishsea_cmd = config["wave forecasts"]["salishsea cmd"] defns = ( f'RUN_ID="{ddmmmyy}ww3-{run_type}"\n' f'WORK_DIR="{run_dir_path}"\n' f'RESULTS_DIR="{results_path/ddmmmyy}"\n' f'WW3_EXE="{wwatch3_exe_path}"\n' - f'MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile ${{HOME}}/mpi_hosts"\n' + f'MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile {mpi_hosts_file}"\n' f'GATHER="{salishsea_cmd} gather"\n' ) return defns diff --git a/tests/test_next_workers.py b/tests/test_next_workers.py index 35c3c29d..cbb97d71 100644 --- a/tests/test_next_workers.py +++ b/tests/test_next_workers.py @@ -127,14 +127,6 @@ def checklist(): return {} -class TestConfig: - """Unit tests for production YAML config file elements related to next_workers.""" - - def test_wave_forecasts_run_when(self, prod_config): - wave_fcsts = prod_config["wave forecasts"] - assert wave_fcsts["run when"] == "after nowcast-green" - - class TestAfterWorkerFunctions: """Unit test to confirm that all worker modules have a corresponding after_*() function in the next_workers module. @@ -1121,8 +1113,6 @@ def test_success_nowcast(self, config, checklist): def test_success_forecast_launch_make_ww3_wind_file_forecast( self, config, checklist, monkeypatch ): - """storm surge season case of wwatch3 running after NEMO forecast""" - monkeypatch.setitem(config["wave forecasts"], "run when", "after forecast") workers, race_condition_workers = next_workers.after_watch_NEMO( Message( "watch_NEMO", @@ -1130,7 +1120,7 @@ def test_success_forecast_launch_make_ww3_wind_file_forecast( { "forecast": { "host": "arbutus.cloud", - "run date": "2023-03-16", + "run date": "2023-11-14", "completed": True, } }, @@ -1140,7 +1130,7 @@ def test_success_forecast_launch_make_ww3_wind_file_forecast( ) expected = NextWorker( "nowcast.workers.make_ww3_wind_file", - args=["arbutus.cloud", "forecast", "--run-date", "2023-03-16"], + args=["arbutus.cloud", "forecast", "--run-date", "2023-11-14"], host="arbutus.cloud", ) assert workers[0] == expected @@ -1149,8 +1139,6 @@ def test_success_forecast_launch_make_ww3_wind_file_forecast( def test_success_forecast_launch_make_ww3_current_file_forecast( self, config, checklist, monkeypatch ): - """storm surge season case of wwatch3 running after NEMO forecast""" - monkeypatch.setitem(config["wave forecasts"], "run when", "after forecast") workers, race_condition_workers = next_workers.after_watch_NEMO( Message( "watch_NEMO", @@ -1158,7 +1146,7 @@ def test_success_forecast_launch_make_ww3_current_file_forecast( { "forecast": { "host": "arbutus.cloud", - "run date": "2023-03-16", + "run date": "2023-11-14", "completed": True, } }, @@ -1168,50 +1156,13 @@ def test_success_forecast_launch_make_ww3_current_file_forecast( ) expected = NextWorker( "nowcast.workers.make_ww3_current_file", - args=["arbutus.cloud", "forecast", "--run-date", "2023-03-16"], + args=["arbutus.cloud", "forecast", "--run-date", "2023-11-14"], host="arbutus.cloud", ) assert workers[1] == expected assert race_condition_workers == {"make_ww3_wind_file", "make_ww3_current_file"} - def test_success_forecast_ww3_after_nowcast_green_launch_upload_forcing_turbidity( - self, config, checklist, monkeypatch - ): - workers = next_workers.after_watch_NEMO( - Message( - "watch_NEMO", - "success forecast", - { - "forecast": { - "host": "arbutus.cloud", - "run date": "2023-07-15", - "completed": True, - } - }, - ), - config, - checklist, - ) - - expected = [ - NextWorker( - "nowcast.workers.upload_forcing", - args=["arbutus.cloud", "turbidity", "--run-date", "2023-07-15"], - ), - NextWorker( - "nowcast.workers.upload_forcing", - args=["orcinus", "turbidity", "--run-date", "2023-07-15"], - ), - ] - assert workers[0:2] == expected - not_expected = NextWorker( - "nowcast.workers.upload_forcing", - args=["salish-nowcast", "turbidity"], - host="localhost", - ) - assert not_expected not in workers - - def test_success_forecast_ww3_after_forecast_launch_upload_forcing_turbidity( + def test_success_forecast_launch_upload_forcing_turbidity( self, config, checklist, monkeypatch ): monkeypatch.setitem(config["wave forecasts"], "run when", "after forecast") @@ -1325,64 +1276,10 @@ def test_success_forecast2_launch_make_ww3_current_file_forecast2( assert workers[1] == expected assert race_condition_workers == {"make_ww3_wind_file", "make_ww3_current_file"} - def test_success_nowcast_green_launch_make_ww3_wind_file_forecast( - self, config, checklist, monkeypatch - ): - monkeypatch.setitem(config["wave forecasts"], "run when", "after nowcast-green") - workers, race_condition_workers = next_workers.after_watch_NEMO( - Message( - "watch_NEMO", - "success nowcast-green", - { - "nowcast-green": { - "host": "arbutus.cloud", - "run date": "2023-03-16", - "completed": True, - } - }, - ), - config, - checklist, - ) - expected = NextWorker( - "nowcast.workers.make_ww3_wind_file", - args=["arbutus.cloud", "forecast", "--run-date", "2023-03-16"], - host="arbutus.cloud", - ) - assert workers[0] == expected - assert race_condition_workers == {"make_ww3_wind_file", "make_ww3_current_file"} - - def test_success_nowcast_green_launch_make_ww3_current_file_forecast( - self, config, checklist, monkeypatch - ): - monkeypatch.setitem(config["wave forecasts"], "run when", "after nowcast-green") - workers, race_condition_workers = next_workers.after_watch_NEMO( - Message( - "watch_NEMO", - "success nowcast-green", - { - "nowcast-green": { - "host": "arbutus.cloud", - "run date": "2023-03-16", - "completed": True, - } - }, - ), - config, - checklist, - ) - expected = NextWorker( - "nowcast.workers.make_ww3_current_file", - args=["arbutus.cloud", "forecast", "--run-date", "2023-03-16"], - host="arbutus.cloud", - ) - assert workers[1] == expected - assert race_condition_workers == {"make_ww3_wind_file", "make_ww3_current_file"} - def test_success_nowcast_green_launch_make_forcing_links_nowcastp_shared_storage( self, config, checklist ): - workers, race_condition_workers = next_workers.after_watch_NEMO( + workers = next_workers.after_watch_NEMO( Message( "watch_NEMO", "success nowcast-green", @@ -1402,8 +1299,7 @@ def test_success_nowcast_green_launch_make_forcing_links_nowcastp_shared_storage args=["salish", "nowcast+", "--shared-storage"], host="localhost", ) - assert expected in workers - assert race_condition_workers == {"make_ww3_wind_file", "make_ww3_current_file"} + assert workers[0] == expected @pytest.mark.parametrize( "msg", diff --git a/tests/workers/test_make_ww3_current_file.py b/tests/workers/test_make_ww3_current_file.py index d6545d07..30a8e429 100644 --- a/tests/workers/test_make_ww3_current_file.py +++ b/tests/workers/test_make_ww3_current_file.py @@ -163,9 +163,6 @@ def test_wave_forecasts_section(self, prod_config): ) assert wave_forecasts["current file template"] == "SoG_current_{yyyymmdd}.nc" - def test_run_when(self, prod_config): - assert prod_config["wave forecasts"]["run when"] == "after nowcast-green" - def test_run_types_section(self, prod_config): run_types = prod_config["run types"] assert run_types["nowcast"]["mesh mask"] == "mesh_mask201702.nc" @@ -267,7 +264,6 @@ def test_checklist( arrow.get("2017-04-12"), Path("/nemoShare/MEOPAR/SalishSea/"), "SalishSea_1h_{s_yyyymmdd}_{e_yyyymmdd}_grid_{grid}.nc", - "nowcast-green", ), ), ( @@ -276,7 +272,6 @@ def test_checklist( arrow.get("2017-04-12"), Path("/nemoShare/MEOPAR/SalishSea/"), "SalishSea_1h_{s_yyyymmdd}_{e_yyyymmdd}_grid_{grid}.nc", - "nowcast-green", ), ), ], @@ -351,46 +346,14 @@ def test_mesh_mask_dataset( class TestCalcNowcastDatasets: """Unit tests for _calc_nowcast_datasets() function.""" - def test_nowcast_datasets_after_nowcast_green(self, caplog): + def test_calc_nowcast_datasets(self, caplog): run_date = arrow.get("2023-03-16") nemo_dir = Path("/nemoShare/MEOPAR/SalishSea/") nemo_file_tmpl = "SalishSea_1h_{s_yyyymmdd}_{e_yyyymmdd}_grid_{grid}.nc" - wave_forecast_after = "nowcast-green" caplog.set_level("DEBUG") datasets = make_ww3_current_file._calc_nowcast_datasets( - run_date, nemo_dir, nemo_file_tmpl, wave_forecast_after - ) - - assert datasets == { - "u": [ - Path( - "/nemoShare/MEOPAR/SalishSea/nowcast-green/16mar23/SalishSea_1h_20230316_20230316_grid_U.nc" - ) - ], - "v": [ - Path( - "/nemoShare/MEOPAR/SalishSea/nowcast-green/16mar23/SalishSea_1h_20230316_20230316_grid_V.nc" - ) - ], - } - expected = [ - "u dataset: /nemoShare/MEOPAR/SalishSea/nowcast-green/16mar23/SalishSea_1h_20230316_20230316_grid_U.nc", - "v dataset: /nemoShare/MEOPAR/SalishSea/nowcast-green/16mar23/SalishSea_1h_20230316_20230316_grid_V.nc", - ] - for i in range(len(expected)): - assert caplog.records[i].levelname == "DEBUG" - assert caplog.messages[i] == expected[i] - - def test_nowcast_datasets_after_forecast(self, caplog): - run_date = arrow.get("2023-03-16") - nemo_dir = Path("/nemoShare/MEOPAR/SalishSea/") - nemo_file_tmpl = "SalishSea_1h_{s_yyyymmdd}_{e_yyyymmdd}_grid_{grid}.nc" - wave_forecast_after = "forecast" - caplog.set_level("DEBUG") - - datasets = make_ww3_current_file._calc_nowcast_datasets( - run_date, nemo_dir, nemo_file_tmpl, wave_forecast_after + run_date, nemo_dir, nemo_file_tmpl ) assert datasets == { @@ -417,54 +380,14 @@ def test_nowcast_datasets_after_forecast(self, caplog): class TestCalcForecastDatasets: """Unit tests for _calc_forecast_datasets() function.""" - def test_forecast_datasets_after_nowcast_green(self, caplog): - run_date = arrow.get("2023-03-16") - nemo_dir = Path("/nemoShare/MEOPAR/SalishSea/") - nemo_file_tmpl = "SalishSea_1h_{s_yyyymmdd}_{e_yyyymmdd}_grid_{grid}.nc" - wave_forecast_after = "nowcast-green" - caplog.set_level("DEBUG") - - datasets = make_ww3_current_file._calc_forecast_datasets( - run_date, nemo_dir, nemo_file_tmpl, wave_forecast_after - ) - - assert datasets == { - "u": [ - Path( - "/nemoShare/MEOPAR/SalishSea/nowcast-green/16mar23/SalishSea_1h_20230316_20230316_grid_U.nc" - ), - Path( - "/nemoShare/MEOPAR/SalishSea/forecast/16mar23/SalishSea_1h_20230317_20230318_grid_U.nc" - ), - ], - "v": [ - Path( - "/nemoShare/MEOPAR/SalishSea/nowcast-green/16mar23/SalishSea_1h_20230316_20230316_grid_V.nc" - ), - Path( - "/nemoShare/MEOPAR/SalishSea/forecast/16mar23/SalishSea_1h_20230317_20230318_grid_V.nc" - ), - ], - } - expected = [ - "u dataset: /nemoShare/MEOPAR/SalishSea/nowcast-green/16mar23/SalishSea_1h_20230316_20230316_grid_U.nc", - "v dataset: /nemoShare/MEOPAR/SalishSea/nowcast-green/16mar23/SalishSea_1h_20230316_20230316_grid_V.nc", - "u dataset: /nemoShare/MEOPAR/SalishSea/forecast/16mar23/SalishSea_1h_20230317_20230318_grid_U.nc", - "v dataset: /nemoShare/MEOPAR/SalishSea/forecast/16mar23/SalishSea_1h_20230317_20230318_grid_V.nc", - ] - for i in range(len(expected)): - assert caplog.records[i].levelname == "DEBUG" - assert caplog.messages[i] == expected[i] - - def test_forecast_datasets_after_forecast(self, caplog): + def test_calc_forecast_datasets(self, caplog): run_date = arrow.get("2023-03-16") nemo_dir = Path("/nemoShare/MEOPAR/SalishSea/") nemo_file_tmpl = "SalishSea_1h_{s_yyyymmdd}_{e_yyyymmdd}_grid_{grid}.nc" - wave_forecast_after = "forecast" caplog.set_level("DEBUG") datasets = make_ww3_current_file._calc_forecast_datasets( - run_date, nemo_dir, nemo_file_tmpl, wave_forecast_after + run_date, nemo_dir, nemo_file_tmpl ) assert datasets == { diff --git a/tests/workers/test_make_ww3_wind_file.py b/tests/workers/test_make_ww3_wind_file.py index 6bb8608f..f6674573 100644 --- a/tests/workers/test_make_ww3_wind_file.py +++ b/tests/workers/test_make_ww3_wind_file.py @@ -226,8 +226,29 @@ def test_nowcast_dataset( m_open_dataset.assert_called_once_with( Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/ops_y2019m03d24.nc") ) + chunks = { + "time_counter": 1, + "y": 230, + "x": 190, + } + drop_vars = { + "LHTFL_surface", + "PRATE_surface", + "RH_2maboveground", + "atmpres", + "precip", + "qair", + "solar", + "tair", + "therm_rad", + } m_open_mfdataset.assert_called_once_with( - [Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/ops_y2019m03d24.nc")] + [Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/ops_y2019m03d24.nc")], + chunks=chunks, + compat="override", + coords="minimal", + data_vars="minimal", + drop_variables=drop_vars, ) def test_forecast_datasets( @@ -242,12 +263,33 @@ def test_forecast_datasets( m_open_dataset.assert_called_once_with( Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/ops_y2017m04d07.nc") ) + chunks = { + "time_counter": 1, + "y": 230, + "x": 190, + } + drop_vars = { + "LHTFL_surface", + "PRATE_surface", + "RH_2maboveground", + "atmpres", + "precip", + "qair", + "solar", + "tair", + "therm_rad", + } m_open_mfdataset.assert_called_once_with( [ Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/ops_y2017m04d07.nc"), Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/fcst/ops_y2017m04d08.nc"), Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/fcst/ops_y2017m04d09.nc"), - ] + ], + chunks=chunks, + compat="override", + coords="minimal", + data_vars="minimal", + drop_variables=drop_vars, ) def test_forecast2_datasets( @@ -262,10 +304,31 @@ def test_forecast2_datasets( m_open_dataset.assert_called_once_with( Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/fcst/ops_y2017m04d07.nc") ) + chunks = { + "time_counter": 1, + "y": 230, + "x": 190, + } + drop_vars = { + "LHTFL_surface", + "PRATE_surface", + "RH_2maboveground", + "atmpres", + "precip", + "qair", + "solar", + "tair", + "therm_rad", + } m_open_mfdataset.assert_called_once_with( [ Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/fcst/ops_y2017m04d07.nc"), Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/fcst/ops_y2017m04d08.nc"), Path("/nemoShare/MEOPAR/GEM2.5/ops/NEMO-atmos/fcst/ops_y2017m04d09.nc"), - ] + ], + chunks=chunks, + compat="override", + coords="minimal", + data_vars="minimal", + drop_variables=drop_vars, ) diff --git a/tests/workers/test_run_ww3.py b/tests/workers/test_run_ww3.py index 455daf4f..6c47315e 100644 --- a/tests/workers/test_run_ww3.py +++ b/tests/workers/test_run_ww3.py @@ -51,6 +51,7 @@ def config(base_config, tmp_path): f"""\ wave forecasts: run prep dir: {run_prep_dir} + mpi hosts file: ${{HOME}}/mpi_hosts.wwatch3 wwatch3 exe path: wwatch3-5.16/exe salishsea cmd: salishsea results: @@ -146,6 +147,10 @@ def test_run_prep_dir(self, prod_config): run_prep_dir = prod_config["wave forecasts"]["run prep dir"] assert run_prep_dir == "/nemoShare/MEOPAR/nowcast-sys/wwatch3-runs" + def test_mpi_hosts_file(self, prod_config): + mpi_hosts_file = prod_config["wave forecasts"]["mpi hosts file"] + assert mpi_hosts_file == "${HOME}/mpi_hosts.wwatch3" + def test_wwatch3_exe(self, prod_config): wwatch3_exe_path = prod_config["wave forecasts"]["wwatch3 exe path"] assert wwatch3_exe_path == "/nemoShare/MEOPAR/nowcast-sys/wwatch3-5.16/exe" @@ -500,7 +505,7 @@ def test_definitions(self, run_type, config): WORK_DIR="wwatch3-runs/tmp_run_dir" RESULTS_DIR="wwatch3-{run_type}/29mar17" WW3_EXE="wwatch3-5.16/exe" - MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile ${{HOME}}/mpi_hosts" + MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile ${{HOME}}/mpi_hosts.wwatch3" GATHER="salishsea gather" """