Add get_vfpa_hadcp after 06 weather collection (#283)

* Add get_vfpa_hadcp after 06 weather collection

Collect the previous day's observations from the VFPA HADCP located at the 2nd
Narrows railway bridge early in the morning after the 06Z weather forecast
products have been downloaded. This restores daily collection of the HADCP obs
that were inadvertently stopped when we stopped running the VHFR FVCOM model in
Mar-2023.

* Update get_vfpa_hadcp main() function docstring

Remove non-informative "Set up and run the worker." line at the beginning.

re: issue #121

* Change NowcastWorker mock to pytest fixture

Test suite maintenance.

re: issue #81

* Change logging mocks to pytest caplog fixture

Replace unittest.mock.patch decorator with pytest caplog fixture for tests of
logging.

Test suite maintenance re: issue #82.

* Modernize unit tests for get_vfpa_hadcp worker

Modified the unit tests in test_get_vfpa_hadcp.py to use pytest fixtures and
monkeypatching for more accurate and isolated tests. This includes new mocks
for make_hour_dataset and write_netcdf, as well as changes in logging and
managing file paths. Particular attention given was given to ensuring accurate
capture of log messages.

nowcast/next_workers.py

@@ -78,6 +78,11 @@ def after_download_weather(msg, config, checklist):
                 next_workers["success 2.5km 06"].append(
                     NextWorker("nowcast.workers.get_onc_ferry", args=[ferry])
                 )
+            next_workers["success 2.5km 06"].append(
+                NextWorker(
+                    "nowcast.workers.get_vfpa_hadcp", args=["--data-date", data_date]
+                )
+            )
             if "forecast2" in config["run types"]:
                 next_workers["success 2.5km 06"].append(
                     NextWorker("nowcast.workers.collect_NeahBay_ssh", args=["00"]),

nowcast/workers/get_vfpa_hadcp.py

@@ -37,9 +37,7 @@
 
 
 def main():
-    """Set up and run the worker.
-
-    For command-line usage see:
+    """For command-line usage see:
 
     :command:`python -m nowcast.workers.get_vfpa_hadcp --help`
     """
@@ -51,6 +49,7 @@ def main():
         help="UTC date to get VFPA HADPC data for.",
     )
     worker.run(get_vfpa_hadcp, success, failure)
+    return worker
 
 
 def success(parsed_args):

tests/test_next_workers.py

@@ -208,6 +208,11 @@ def mock_now():
             NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"),
             NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"),
             NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"),
+            NextWorker(
+                "nowcast.workers.get_vfpa_hadcp",
+                ["--data-date", "2018-12-26"],
+                host="localhost",
+            ),
             NextWorker("nowcast.workers.collect_NeahBay_ssh", ["00"], host="localhost"),
         ]
         assert workers == expected
@@ -327,6 +332,11 @@ def mock_now():
             NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"),
             NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"),
             NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"),
+            NextWorker(
+                "nowcast.workers.get_vfpa_hadcp",
+                ["--data-date", "2018-12-26"],
+                host="localhost",
+            ),
             NextWorker("nowcast.workers.collect_NeahBay_ssh", ["00"], host="localhost"),
             NextWorker(
                 "nowcast.workers.collect_weather", ["12", "2.5km"], host="localhost"

tests/workers/test_get_vfpa_hadcp.py

@@ -18,14 +18,16 @@
 
 """Unit tests for SalishSeaCast get_vfpa_hadcp worker.
 """
+import logging
+import os
 import textwrap
 from pathlib import Path
 from types import SimpleNamespace
-from unittest.mock import Mock, patch
 
 import arrow
 import nemo_nowcast
 import pytest
+import xarray
 
 from nowcast.workers import get_vfpa_hadcp
 
@@ -51,40 +53,30 @@ def config(base_config):
     return config_
 
 
-@patch("nowcast.workers.get_vfpa_hadcp.NowcastWorker", spec=True)
+@pytest.fixture
+def mock_worker(mock_nowcast_worker, monkeypatch):
+    monkeypatch.setattr(get_vfpa_hadcp, "NowcastWorker", mock_nowcast_worker)
+
+
 class TestMain:
     """Unit tests for main() function."""
 
-    def test_instantiate_worker(self, m_worker):
-        m_worker().cli = Mock(name="cli")
-        get_vfpa_hadcp.main()
-        args, kwargs = m_worker.call_args
-        assert args == ("get_vfpa_hadcp",)
-        assert list(kwargs.keys()) == ["description"]
-
-    def test_init_cli(self, m_worker):
-        m_worker().cli = Mock(name="cli")
-        get_vfpa_hadcp.main()
-        m_worker().init_cli.assert_called_once_with()
-
-    def test_add_data_date_option(self, m_worker):
-        m_worker().cli = Mock(name="cli")
-        get_vfpa_hadcp.main()
-        args, kwargs = m_worker().cli.add_date_option.call_args_list[0]
-        assert args == ("--data-date",)
-        assert kwargs["default"] == arrow.now().floor("day")
-        assert "help" in kwargs
-
-    def test_run_worker(self, m_worker):
-        m_worker().cli = Mock(name="cli")
-        get_vfpa_hadcp.main()
-        args, kwargs = m_worker().run.call_args
-        assert args == (
-            get_vfpa_hadcp.get_vfpa_hadcp,
-            get_vfpa_hadcp.success,
-            get_vfpa_hadcp.failure,
+    def test_instantiate_worker(self, mock_worker):
+        worker = get_vfpa_hadcp.main()
+
+        assert worker.name == "get_vfpa_hadcp"
+        assert worker.description.startswith(
+            "SalishSeaCast worker that processes VFPA HADCP observations from the 2nd Narrows Rail Bridge"
         )
 
+    def test_add_data_date_option(self, mock_worker):
+        worker = get_vfpa_hadcp.main()
+        assert worker.cli.parser._actions[3].dest == "data_date"
+        expected = nemo_nowcast.cli.CommandLineInterface.arrow_date
+        assert worker.cli.parser._actions[3].type == expected
+        assert worker.cli.parser._actions[3].default == arrow.now().floor("day")
+        assert worker.cli.parser._actions[3].help
+
 
 class TestConfig:
     """Unit tests for production YAML config file elements related to worker."""
@@ -107,75 +99,159 @@ def test_observations(self, prod_config):
         assert hadcp_obs["filepath template"] == "VFPA_2ND_NARROWS_HADCP_2s_{yyyymm}.nc"
 
 
-@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True)
 class TestSuccess:
     """Unit test for success() function."""
 
-    def test_success(self, m_logger):
+    def test_success(self, caplog):
         parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01"))
+        caplog.set_level(logging.DEBUG)
         msg_type = get_vfpa_hadcp.success(parsed_args)
-        m_logger.info.assert_called_once_with(
-            "VFPA HADCP observations added to 2018-10 netcdf file"
-        )
+        assert caplog.records[0].levelname == "INFO"
+        expected = "VFPA HADCP observations added to 2018-10 netcdf file"
+        assert caplog.messages[0] == expected
         assert msg_type == "success"
 
 
-@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True)
 class TestFailure:
     """Unit test for failure() function."""
 
-    def test_failure(self, m_logger):
+    def test_failure(self, caplog):
         parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01"))
+        caplog.set_level(logging.DEBUG)
         msg_type = get_vfpa_hadcp.failure(parsed_args)
-        m_logger.critical.assert_called_once_with(
-            "Addition of VFPA HADCP observations to 2018-10 netcdf file failed"
-        )
+        assert caplog.records[0].levelname == "CRITICAL"
+        expected = "Addition of VFPA HADCP observations to 2018-10 netcdf file failed"
+        assert caplog.messages[0] == expected
         assert msg_type == "failure"
 
 
-@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True)
-@patch("nowcast.workers.get_vfpa_hadcp._make_hour_dataset", autospec=True)
 class TestGetVFPA_HADCP:
     """Unit test for get_vfpa_hadcp() function."""
 
-    def test_checklist_create(self, m_mk_hr_ds, m_logger, config):
+    @staticmethod
+    @pytest.fixture
+    def mock_make_hour_dataset(monkeypatch):
+
+        def _mock_make_hour_dataset(csv_dir, utc_start_hr, place):
+            return xarray.Dataset()
+
+        monkeypatch.setattr(
+            get_vfpa_hadcp, "_make_hour_dataset", _mock_make_hour_dataset
+        )
+
+    @staticmethod
+    @pytest.fixture
+    def mock_write_netcdf(monkeypatch):
+        def _mock_write_netcdf(ds, nc_filepath):
+            return
+
+        monkeypatch.setattr(get_vfpa_hadcp, "_write_netcdf", _mock_write_netcdf)
+
+    @pytest.mark.parametrize("nc_file_exists", (True, False))
+    def test_log_messages(
+        self,
+        nc_file_exists,
+        mock_make_hour_dataset,
+        mock_write_netcdf,
+        config,
+        caplog,
+        tmp_path,
+        monkeypatch,
+    ):
+        dest_dir = tmp_path
+        monkeypatch.setitem(
+            config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir)
+        )
+        nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_202407.nc"
+        if nc_file_exists:
+            nc_filepath.write_bytes(b"")
+        parsed_args = SimpleNamespace(data_date=arrow.get("2024-07-13"))
+        caplog.set_level(logging.DEBUG)
+        get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
+        assert caplog.records[0].levelname == "INFO"
+        expected = (
+            "processing VFPA HADCP data from 2nd Narrows Rail Bridge for 2024-07-13"
+        )
+        assert caplog.messages[0] == expected
+        if not nc_file_exists:
+            assert caplog.records[1].levelname == "INFO"
+            assert caplog.records[1].message.startswith("created")
+            assert caplog.messages[1].endswith("VFPA_2ND_NARROWS_HADCP_2s_202407.nc")
+        for rec_num, hr in zip(range(2, 24), range(1, 23)):
+            assert caplog.records[rec_num].levelname == "DEBUG"
+            expected = f"no data for 2024-07-13 {hr:02d}:00 hour"
+            assert caplog.messages[rec_num] == expected
+        assert caplog.records[25].levelname == "INFO"
+        expected = f"added VFPA HADCP data from 2nd Narrows Rail Bridge for 2024-07-13 to {nc_filepath}"
+        assert caplog.messages[25] == expected
+
+    def test_checklist_create(
+        self,
+        mock_make_hour_dataset,
+        mock_write_netcdf,
+        config,
+        caplog,
+        tmp_path,
+        monkeypatch,
+    ):
+        dest_dir = tmp_path
+        monkeypatch.setitem(
+            config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir)
+        )
+        nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201810.nc"
         parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01"))
+        caplog.set_level(logging.DEBUG)
         checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
         expected = {
-            "created": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201810.nc",
+            "created": f"{nc_filepath}",
             "UTC date": "2018-10-01",
         }
         assert checklist == expected
 
-    @patch(
-        "nowcast.workers.get_vfpa_hadcp.Path.exists", return_value=True, autospec=True
-    )
-    @patch("nowcast.workers.get_vfpa_hadcp.xarray", autospec=True)
-    def test_checklist_extend(self, m_xarray, m_exists, m_mk_hr_ds, m_logger, config):
+    def test_checklist_extend(
+        self,
+        mock_make_hour_dataset,
+        mock_write_netcdf,
+        config,
+        caplog,
+        tmp_path,
+        monkeypatch,
+    ):
+        dest_dir = tmp_path
+        monkeypatch.setitem(
+            config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir)
+        )
+        nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201810.nc"
+        xarray.DataArray().to_netcdf(nc_filepath)
         parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-21"))
+        caplog.set_level(logging.DEBUG)
         checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
         expected = {
-            "extended": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201810.nc",
+            "extended": f"{nc_filepath}",
             "UTC date": "2018-10-21",
         }
         assert checklist == expected
 
-    @pytest.mark.parametrize("ds_exists", (True, False))
-    @patch("nowcast.workers.get_vfpa_hadcp.xarray", autospec=True)
     def test_checklist_missing_data(
-        self, m_xarray, m_mk_hr_ds, m_logger, ds_exists, config
+        self,
+        mock_make_hour_dataset,
+        mock_write_netcdf,
+        config,
+        caplog,
+        tmp_path,
+        monkeypatch,
     ):
-        parsed_args = SimpleNamespace(data_date=arrow.get("2018-12-23"))
-        m_mk_hr_ds.side_effect = ValueError
-        p_exists = patch(
-            "nowcast.workers.get_vfpa_hadcp.Path.exists",
-            return_value=ds_exists,
-            autospec=True,
+        dest_dir = tmp_path
+        monkeypatch.setitem(
+            config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir)
         )
-        with p_exists:
-            checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
+        nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201812.nc"
+        nc_filepath.write_bytes(b"")
+        caplog.set_level(logging.DEBUG)
+        parsed_args = SimpleNamespace(data_date=arrow.get("2018-12-23"))
+        checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
         expected = {
-            "missing data": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201812.nc",
+            "missing data": f"{nc_filepath}",
             "UTC date": "2018-12-23",
         }
         assert checklist == expected