Skip to content

Commit

Permalink
Collect USGS rivers day-avg flow after 12Z weather
Browse files Browse the repository at this point in the history 
Because production tests show that values are not available at PST time when
[collect|download]_weather 06 runs.
  • Loading branch information
@douglatornell
douglatornell committed Jan 10, 2023
1 parent 3bb8673 commit 848650a
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 36 deletions.
22 changes: 14 additions & 8 deletions nowcast/next_workers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,15 @@ def after_download_weather(msg, config, checklist):
"success 1km 12": [],
}
if msg.type.startswith("success"):
data_date = arrow.now().shift(days=-1).format("YYYY-MM-DD")
if msg.type.endswith("2.5km 06"):
data_date = arrow.now().shift(days=-1).format("YYYY-MM-DD")
for data_src in config["rivers"]["stations"]:
for river_name in config["rivers"]["stations"][data_src]:
next_workers["success 2.5km 06"].append(
NextWorker(
"nowcast.workers.collect_river_data",
args=[data_src, river_name, "--data-date", data_date],
)
for river_name in config["rivers"]["stations"]["ECCC"]:
next_workers["success 2.5km 06"].append(
NextWorker(
"nowcast.workers.collect_river_data",
args=["ECCC", river_name, "--data-date", data_date],
)
)
for stn in config["observations"]["ctd data"]["stations"]:
next_workers["success 2.5km 06"].append(
NextWorker("nowcast.workers.get_onc_ctd", args=[stn])
Expand All @@ -89,6 +88,13 @@ def after_download_weather(msg, config, checklist):
race_condition_workers = {"grib_to_netcdf", "make_ssh_files"}
return next_workers[msg.type], race_condition_workers
if msg.type.endswith("2.5km 12"):
for river_name in config["rivers"]["stations"]["USGS"]:
next_workers["success 2.5km 12"].append(
NextWorker(
"nowcast.workers.collect_river_data",
args=["USGS", river_name, "--data-date", data_date],
)
)
next_workers["success 2.5km 12"].extend(
[
NextWorker("nowcast.workers.collect_NeahBay_ssh", args=["06"]),
Expand Down
52 changes: 24 additions & 28 deletions tests/test_next_workers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,11 +186,6 @@ def mock_now():

monkeypatch.setattr(next_workers.arrow, "now", mock_now)

def mock_utcnow():
return arrow.get("2018-12-27")

monkeypatch.setattr(next_workers.arrow, "utcnow", mock_utcnow)

workers, race_condition_workers = next_workers.after_download_weather(
Message("download_weather", "success 2.5km 06"), config, checklist
)
Expand All @@ -210,11 +205,6 @@ def mock_utcnow():
["ECCC", "Fraser", "--data-date", "2018-12-26"],
host="localhost",
),
NextWorker(
"nowcast.workers.collect_river_data",
["USGS", "SkagitMountVernon", "--data-date", "2018-12-26"],
host="localhost",
),
NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"),
NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"),
NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"),
Expand All @@ -226,20 +216,28 @@ def mock_utcnow():
assert workers == expected
assert race_condition_workers == {"grib_to_netcdf", "make_ssh_files"}

def test_success_2_5_km_12(self, config, checklist):
def test_success_2_5_km_12(self, config, checklist, monkeypatch):
def mock_now():
return arrow.get("2018-12-27")

monkeypatch.setattr(next_workers.arrow, "now", mock_now)

workers, race_condition_workers = next_workers.after_download_weather(
Message("download_weather", "success 2.5km 12"), config, checklist
)
expected = [
NextWorker(
"nowcast.workers.collect_river_data",
["USGS", "SkagitMountVernon", "--data-date", "2018-12-26"],
host="localhost",
),
NextWorker("nowcast.workers.collect_NeahBay_ssh", ["06"], host="localhost"),
NextWorker(
"nowcast.workers.grib_to_netcdf", ["nowcast+"], host="localhost"
),
NextWorker("nowcast.workers.download_live_ocean", [], host="localhost"),
]
assert len(workers) == len(expected)
for next_worker in expected:
assert next_worker in workers
assert workers == expected
assert race_condition_workers == {
"grib_to_netcdf",
"make_live_ocean_files",
Expand Down Expand Up @@ -274,11 +272,6 @@ def mock_now():

monkeypatch.setattr(next_workers.arrow, "now", mock_now)

def mock_utcnow():
return arrow.get("2018-12-27")

monkeypatch.setattr(next_workers.arrow, "utcnow", mock_utcnow)

workers, race_condition_workers = next_workers.after_collect_weather(
Message("collect_weather", "success 2.5km 06"), config, checklist
)
Expand All @@ -298,11 +291,6 @@ def mock_utcnow():
["ECCC", "Fraser", "--data-date", "2018-12-26"],
host="localhost",
),
NextWorker(
"nowcast.workers.collect_river_data",
["USGS", "SkagitMountVernon", "--data-date", "2018-12-26"],
host="localhost",
),
NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"),
NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"),
NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"),
Expand All @@ -317,11 +305,21 @@ def mock_utcnow():
assert workers == expected
assert race_condition_workers == {"grib_to_netcdf", "make_ssh_files"}

def test_success_2_5_km_12(self, config, checklist):
def test_success_2_5_km_12(self, config, checklist, monkeypatch):
def mock_now():
return arrow.get("2018-12-27")

monkeypatch.setattr(next_workers.arrow, "now", mock_now)

workers, race_condition_workers = next_workers.after_collect_weather(
Message("collect_weather", "success 2.5km 12"), config, checklist
)
expected = [
NextWorker(
"nowcast.workers.collect_river_data",
["USGS", "SkagitMountVernon", "--data-date", "2018-12-26"],
host="localhost",
),
NextWorker("nowcast.workers.collect_NeahBay_ssh", ["06"], host="localhost"),
NextWorker(
"nowcast.workers.grib_to_netcdf", ["nowcast+"], host="localhost"
Expand All @@ -331,9 +329,7 @@ def test_success_2_5_km_12(self, config, checklist):
"nowcast.workers.collect_weather", ["18", "2.5km"], host="localhost"
),
]
assert len(workers) == len(expected)
for next_worker in expected:
assert next_worker in workers
assert workers == expected
assert race_condition_workers == {
"grib_to_netcdf",
"make_live_ocean_files",
Expand Down

0 comments on commit 848650a

Please sign in to comment.