From 5261aa16aad5b09ec326ad8af3e7f9e25c94c171 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Tue, 16 Jul 2024 16:20:52 -0700 Subject: [PATCH] Add get_vfpa_hadcp after 06 weather collection (#283) * Add get_vfpa_hadcp after 06 weather collection Collect the previous day's observations from the VFPA HADCP located at the 2nd Narrows railway bridge early in the morning after the 06Z weather forecast products have been downloaded. This restores daily collection of the HADCP obs that were inadvertently stopped when we stopped running the VHFR FVCOM model in Mar-2023. * Update get_vfpa_hadcp main() function docstring Remove non-informative "Set up and run the worker." line at the beginning. re: issue #121 * Change NowcastWorker mock to pytest fixture Test suite maintenance. re: issue #81 * Change logging mocks to pytest caplog fixture Replace unittest.mock.patch decorator with pytest caplog fixture for tests of logging. Test suite maintenance re: issue #82. * Modernize unit tests for get_vfpa_hadcp worker Modified the unit tests in test_get_vfpa_hadcp.py to use pytest fixtures and monkeypatching for more accurate and isolated tests. This includes new mocks for make_hour_dataset and write_netcdf, as well as changes in logging and managing file paths. Particular attention given was given to ensuring accurate capture of log messages. --- nowcast/next_workers.py | 5 + nowcast/workers/get_vfpa_hadcp.py | 5 +- tests/test_next_workers.py | 10 ++ tests/workers/test_get_vfpa_hadcp.py | 200 ++++++++++++++++++--------- 4 files changed, 155 insertions(+), 65 deletions(-) diff --git a/nowcast/next_workers.py b/nowcast/next_workers.py index 56bebd02..6fb0d59f 100644 --- a/nowcast/next_workers.py +++ b/nowcast/next_workers.py @@ -78,6 +78,11 @@ def after_download_weather(msg, config, checklist): next_workers["success 2.5km 06"].append( NextWorker("nowcast.workers.get_onc_ferry", args=[ferry]) ) + next_workers["success 2.5km 06"].append( + NextWorker( + "nowcast.workers.get_vfpa_hadcp", args=["--data-date", data_date] + ) + ) if "forecast2" in config["run types"]: next_workers["success 2.5km 06"].append( NextWorker("nowcast.workers.collect_NeahBay_ssh", args=["00"]), diff --git a/nowcast/workers/get_vfpa_hadcp.py b/nowcast/workers/get_vfpa_hadcp.py index e2e34188..0c6d53ad 100644 --- a/nowcast/workers/get_vfpa_hadcp.py +++ b/nowcast/workers/get_vfpa_hadcp.py @@ -37,9 +37,7 @@ def main(): - """Set up and run the worker. - - For command-line usage see: + """For command-line usage see: :command:`python -m nowcast.workers.get_vfpa_hadcp --help` """ @@ -51,6 +49,7 @@ def main(): help="UTC date to get VFPA HADPC data for.", ) worker.run(get_vfpa_hadcp, success, failure) + return worker def success(parsed_args): diff --git a/tests/test_next_workers.py b/tests/test_next_workers.py index e535214e..d5589997 100644 --- a/tests/test_next_workers.py +++ b/tests/test_next_workers.py @@ -208,6 +208,11 @@ def mock_now(): NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"), NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"), NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"), + NextWorker( + "nowcast.workers.get_vfpa_hadcp", + ["--data-date", "2018-12-26"], + host="localhost", + ), NextWorker("nowcast.workers.collect_NeahBay_ssh", ["00"], host="localhost"), ] assert workers == expected @@ -327,6 +332,11 @@ def mock_now(): NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"), NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"), NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"), + NextWorker( + "nowcast.workers.get_vfpa_hadcp", + ["--data-date", "2018-12-26"], + host="localhost", + ), NextWorker("nowcast.workers.collect_NeahBay_ssh", ["00"], host="localhost"), NextWorker( "nowcast.workers.collect_weather", ["12", "2.5km"], host="localhost" diff --git a/tests/workers/test_get_vfpa_hadcp.py b/tests/workers/test_get_vfpa_hadcp.py index b91cb87a..b95ab9fd 100644 --- a/tests/workers/test_get_vfpa_hadcp.py +++ b/tests/workers/test_get_vfpa_hadcp.py @@ -18,14 +18,16 @@ """Unit tests for SalishSeaCast get_vfpa_hadcp worker. """ +import logging +import os import textwrap from pathlib import Path from types import SimpleNamespace -from unittest.mock import Mock, patch import arrow import nemo_nowcast import pytest +import xarray from nowcast.workers import get_vfpa_hadcp @@ -51,40 +53,30 @@ def config(base_config): return config_ -@patch("nowcast.workers.get_vfpa_hadcp.NowcastWorker", spec=True) +@pytest.fixture +def mock_worker(mock_nowcast_worker, monkeypatch): + monkeypatch.setattr(get_vfpa_hadcp, "NowcastWorker", mock_nowcast_worker) + + class TestMain: """Unit tests for main() function.""" - def test_instantiate_worker(self, m_worker): - m_worker().cli = Mock(name="cli") - get_vfpa_hadcp.main() - args, kwargs = m_worker.call_args - assert args == ("get_vfpa_hadcp",) - assert list(kwargs.keys()) == ["description"] - - def test_init_cli(self, m_worker): - m_worker().cli = Mock(name="cli") - get_vfpa_hadcp.main() - m_worker().init_cli.assert_called_once_with() - - def test_add_data_date_option(self, m_worker): - m_worker().cli = Mock(name="cli") - get_vfpa_hadcp.main() - args, kwargs = m_worker().cli.add_date_option.call_args_list[0] - assert args == ("--data-date",) - assert kwargs["default"] == arrow.now().floor("day") - assert "help" in kwargs - - def test_run_worker(self, m_worker): - m_worker().cli = Mock(name="cli") - get_vfpa_hadcp.main() - args, kwargs = m_worker().run.call_args - assert args == ( - get_vfpa_hadcp.get_vfpa_hadcp, - get_vfpa_hadcp.success, - get_vfpa_hadcp.failure, + def test_instantiate_worker(self, mock_worker): + worker = get_vfpa_hadcp.main() + + assert worker.name == "get_vfpa_hadcp" + assert worker.description.startswith( + "SalishSeaCast worker that processes VFPA HADCP observations from the 2nd Narrows Rail Bridge" ) + def test_add_data_date_option(self, mock_worker): + worker = get_vfpa_hadcp.main() + assert worker.cli.parser._actions[3].dest == "data_date" + expected = nemo_nowcast.cli.CommandLineInterface.arrow_date + assert worker.cli.parser._actions[3].type == expected + assert worker.cli.parser._actions[3].default == arrow.now().floor("day") + assert worker.cli.parser._actions[3].help + class TestConfig: """Unit tests for production YAML config file elements related to worker.""" @@ -107,75 +99,159 @@ def test_observations(self, prod_config): assert hadcp_obs["filepath template"] == "VFPA_2ND_NARROWS_HADCP_2s_{yyyymm}.nc" -@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True) class TestSuccess: """Unit test for success() function.""" - def test_success(self, m_logger): + def test_success(self, caplog): parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01")) + caplog.set_level(logging.DEBUG) msg_type = get_vfpa_hadcp.success(parsed_args) - m_logger.info.assert_called_once_with( - "VFPA HADCP observations added to 2018-10 netcdf file" - ) + assert caplog.records[0].levelname == "INFO" + expected = "VFPA HADCP observations added to 2018-10 netcdf file" + assert caplog.messages[0] == expected assert msg_type == "success" -@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True) class TestFailure: """Unit test for failure() function.""" - def test_failure(self, m_logger): + def test_failure(self, caplog): parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01")) + caplog.set_level(logging.DEBUG) msg_type = get_vfpa_hadcp.failure(parsed_args) - m_logger.critical.assert_called_once_with( - "Addition of VFPA HADCP observations to 2018-10 netcdf file failed" - ) + assert caplog.records[0].levelname == "CRITICAL" + expected = "Addition of VFPA HADCP observations to 2018-10 netcdf file failed" + assert caplog.messages[0] == expected assert msg_type == "failure" -@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True) -@patch("nowcast.workers.get_vfpa_hadcp._make_hour_dataset", autospec=True) class TestGetVFPA_HADCP: """Unit test for get_vfpa_hadcp() function.""" - def test_checklist_create(self, m_mk_hr_ds, m_logger, config): + @staticmethod + @pytest.fixture + def mock_make_hour_dataset(monkeypatch): + + def _mock_make_hour_dataset(csv_dir, utc_start_hr, place): + return xarray.Dataset() + + monkeypatch.setattr( + get_vfpa_hadcp, "_make_hour_dataset", _mock_make_hour_dataset + ) + + @staticmethod + @pytest.fixture + def mock_write_netcdf(monkeypatch): + def _mock_write_netcdf(ds, nc_filepath): + return + + monkeypatch.setattr(get_vfpa_hadcp, "_write_netcdf", _mock_write_netcdf) + + @pytest.mark.parametrize("nc_file_exists", (True, False)) + def test_log_messages( + self, + nc_file_exists, + mock_make_hour_dataset, + mock_write_netcdf, + config, + caplog, + tmp_path, + monkeypatch, + ): + dest_dir = tmp_path + monkeypatch.setitem( + config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir) + ) + nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_202407.nc" + if nc_file_exists: + nc_filepath.write_bytes(b"") + parsed_args = SimpleNamespace(data_date=arrow.get("2024-07-13")) + caplog.set_level(logging.DEBUG) + get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) + assert caplog.records[0].levelname == "INFO" + expected = ( + "processing VFPA HADCP data from 2nd Narrows Rail Bridge for 2024-07-13" + ) + assert caplog.messages[0] == expected + if not nc_file_exists: + assert caplog.records[1].levelname == "INFO" + assert caplog.records[1].message.startswith("created") + assert caplog.messages[1].endswith("VFPA_2ND_NARROWS_HADCP_2s_202407.nc") + for rec_num, hr in zip(range(2, 24), range(1, 23)): + assert caplog.records[rec_num].levelname == "DEBUG" + expected = f"no data for 2024-07-13 {hr:02d}:00 hour" + assert caplog.messages[rec_num] == expected + assert caplog.records[25].levelname == "INFO" + expected = f"added VFPA HADCP data from 2nd Narrows Rail Bridge for 2024-07-13 to {nc_filepath}" + assert caplog.messages[25] == expected + + def test_checklist_create( + self, + mock_make_hour_dataset, + mock_write_netcdf, + config, + caplog, + tmp_path, + monkeypatch, + ): + dest_dir = tmp_path + monkeypatch.setitem( + config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir) + ) + nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201810.nc" parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01")) + caplog.set_level(logging.DEBUG) checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) expected = { - "created": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201810.nc", + "created": f"{nc_filepath}", "UTC date": "2018-10-01", } assert checklist == expected - @patch( - "nowcast.workers.get_vfpa_hadcp.Path.exists", return_value=True, autospec=True - ) - @patch("nowcast.workers.get_vfpa_hadcp.xarray", autospec=True) - def test_checklist_extend(self, m_xarray, m_exists, m_mk_hr_ds, m_logger, config): + def test_checklist_extend( + self, + mock_make_hour_dataset, + mock_write_netcdf, + config, + caplog, + tmp_path, + monkeypatch, + ): + dest_dir = tmp_path + monkeypatch.setitem( + config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir) + ) + nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201810.nc" + xarray.DataArray().to_netcdf(nc_filepath) parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-21")) + caplog.set_level(logging.DEBUG) checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) expected = { - "extended": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201810.nc", + "extended": f"{nc_filepath}", "UTC date": "2018-10-21", } assert checklist == expected - @pytest.mark.parametrize("ds_exists", (True, False)) - @patch("nowcast.workers.get_vfpa_hadcp.xarray", autospec=True) def test_checklist_missing_data( - self, m_xarray, m_mk_hr_ds, m_logger, ds_exists, config + self, + mock_make_hour_dataset, + mock_write_netcdf, + config, + caplog, + tmp_path, + monkeypatch, ): - parsed_args = SimpleNamespace(data_date=arrow.get("2018-12-23")) - m_mk_hr_ds.side_effect = ValueError - p_exists = patch( - "nowcast.workers.get_vfpa_hadcp.Path.exists", - return_value=ds_exists, - autospec=True, + dest_dir = tmp_path + monkeypatch.setitem( + config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir) ) - with p_exists: - checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) + nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201812.nc" + nc_filepath.write_bytes(b"") + caplog.set_level(logging.DEBUG) + parsed_args = SimpleNamespace(data_date=arrow.get("2018-12-23")) + checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) expected = { - "missing data": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201812.nc", + "missing data": f"{nc_filepath}", "UTC date": "2018-12-23", } assert checklist == expected