From 583968427852b15955da063f7c65ad1702cbc20b Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Tue, 7 Nov 2023 15:44:50 -0800 Subject: [PATCH] Move wwatch3 prep to after forecast2 result download Testing the hypothesis that the make_ww3_* worker stalls are due to file system overload that results downloading contributes to. --- nowcast/next_workers.py | 59 ++++++++++++----------- tests/test_next_workers.py | 96 +++++++++++++++++--------------------- 2 files changed, 75 insertions(+), 80 deletions(-) diff --git a/nowcast/next_workers.py b/nowcast/next_workers.py index 16c04b26..313b6a22 100644 --- a/nowcast/next_workers.py +++ b/nowcast/next_workers.py @@ -888,34 +888,6 @@ def after_watch_NEMO(msg, config, checklist): args=[host, "nowcast+", "--shared-storage"], ) ) - if run_type == "forecast2": - host_name = config["wave forecasts"]["host"] - run_date = arrow.get(msg.payload[run_type]["run date"]).shift(days=+1) - next_workers[msg.type].extend( - [ - NextWorker( - "nowcast.workers.make_ww3_wind_file", - args=[ - host_name, - "forecast2", - "--run-date", - run_date.format("YYYY-MM-DD"), - ], - host=host_name, - ), - NextWorker( - "nowcast.workers.make_ww3_current_file", - args=[ - host_name, - "forecast2", - "--run-date", - run_date.format("YYYY-MM-DD"), - ], - host=host_name, - ), - ] - ) - race_condition_workers = {"make_ww3_wind_file", "make_ww3_current_file"} enabled_host_config = config["run"]["enabled hosts"][ msg.payload[run_type]["host"] ] @@ -1511,6 +1483,7 @@ def after_download_results(msg, config, checklist): "success hindcast": [], "success nowcast-agrif": [], } + race_condition_workers = {} if msg.type.startswith("success"): run_type = msg.type.split()[1] run_date = msg.payload[run_type]["run date"] @@ -1566,6 +1539,36 @@ def after_download_results(msg, config, checklist): args=[run_type, "--run-date", run_date], ) ) + if run_type == "forecast2": + host_name = config["wave forecasts"]["host"] + run_date = arrow.get(msg.payload[run_type]["run date"]).shift(days=+1) + next_workers[msg.type].extend( + [ + NextWorker( + "nowcast.workers.make_ww3_wind_file", + args=[ + host_name, + "forecast2", + "--run-date", + run_date.format("YYYY-MM-DD"), + ], + host=host_name, + ), + NextWorker( + "nowcast.workers.make_ww3_current_file", + args=[ + host_name, + "forecast2", + "--run-date", + run_date.format("YYYY-MM-DD"), + ], + host=host_name, + ), + ] + ) + race_condition_workers = {"make_ww3_wind_file", "make_ww3_current_file"} + if race_condition_workers: + return next_workers[msg.type], race_condition_workers return next_workers[msg.type] diff --git a/tests/test_next_workers.py b/tests/test_next_workers.py index 35c3c29d..1885d015 100644 --- a/tests/test_next_workers.py +++ b/tests/test_next_workers.py @@ -1273,58 +1273,6 @@ def test_success_forecast_no_launch_make_fvcom_boundary(self, config, checklist) ) assert expected not in workers - def test_success_forecast2_launch_make_ww3_wind_file_forecast2( - self, config, checklist - ): - workers, race_condition_workers = next_workers.after_watch_NEMO( - Message( - "watch_NEMO", - "success forecast2", - { - "forecast2": { - "host": "arbutus.cloud", - "run date": "2023-04-07", - "completed": True, - } - }, - ), - config, - checklist, - ) - expected = NextWorker( - "nowcast.workers.make_ww3_wind_file", - args=["arbutus.cloud", "forecast2", "--run-date", "2023-04-08"], - host="arbutus.cloud", - ) - assert workers[0] == expected - assert race_condition_workers == {"make_ww3_wind_file", "make_ww3_current_file"} - - def test_success_forecast2_launch_make_ww3_current_file_forecast2( - self, config, checklist - ): - workers, race_condition_workers = next_workers.after_watch_NEMO( - Message( - "watch_NEMO", - "success forecast2", - { - "forecast2": { - "host": "arbutus.cloud", - "run date": "2023-04-07", - "completed": True, - } - }, - ), - config, - checklist, - ) - expected = NextWorker( - "nowcast.workers.make_ww3_current_file", - args=["arbutus.cloud", "forecast2", "--run-date", "2023-04-08"], - host="arbutus.cloud", - ) - assert workers[1] == expected - assert race_condition_workers == {"make_ww3_wind_file", "make_ww3_current_file"} - def test_success_nowcast_green_launch_make_ww3_wind_file_forecast( self, config, checklist, monkeypatch ): @@ -2247,6 +2195,46 @@ def test_success_nowcast_launch_make_plots_specials( ) assert expected in workers + def test_success_forecast2_launch_make_ww3_wind_file_forecast2( + self, config, checklist + ): + workers, race_condition_workers = next_workers.after_download_results( + Message( + "download_results", + "success forecast2", + payload={"forecast2": {"run date": "2023-11-07"}}, + ), + config, + checklist, + ) + expected = NextWorker( + "nowcast.workers.make_ww3_wind_file", + args=["arbutus.cloud", "forecast2", "--run-date", "2023-11-08"], + host="arbutus.cloud", + ) + assert workers[1] == expected + assert race_condition_workers == {"make_ww3_wind_file", "make_ww3_current_file"} + + def test_success_forecast2_launch_make_ww3_current_file_forecast2( + self, config, checklist + ): + workers, race_condition_workers = next_workers.after_download_results( + Message( + "watch_NEMO", + "success forecast2", + payload={"forecast2": {"run date": "2023-11-07"}}, + ), + config, + checklist, + ) + expected = NextWorker( + "nowcast.workers.make_ww3_current_file", + args=["arbutus.cloud", "forecast2", "--run-date", "2023-11-08"], + host="arbutus.cloud", + ) + assert workers[2] == expected + assert race_condition_workers == {"make_ww3_wind_file", "make_ww3_current_file"} + def test_success_nowcast_green_launch_ping_erddap_nowcast_green( self, config, checklist ): @@ -2327,6 +2315,10 @@ def test_success_launch_make_CHS_currents_file( args=[run_type, "--run-date", run_date], host="localhost", ) + try: + workers, race_condition_workers = workers + except ValueError: + pass assert expected in workers def test_success_hindcast_launch_split_results(