From 8c0978d2d339580234a4dc7882a66d368f8dbab4 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Apr 2024 10:46:45 +0200 Subject: [PATCH 1/7] remove tmp files --- nwcsafpps_runner/prepare_nwp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 608b20d..1953b66 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -227,6 +227,7 @@ def update_nwp_inner(starttime, nlengths, cfg): for fname in get_files_to_process(cfg): file_obj = NWPFileFamily(cfg, fname) if should_be_skipped(file_obj, starttime, nlengths): + remove_file(file_obj.tmp_filename) continue LOG.debug("Analysis time and start time: {:s} {:s}".format(str(file_obj.analysis_time), str(starttime))) @@ -235,7 +236,6 @@ def update_nwp_inner(starttime, nlengths, cfg): out_file = create_nwp_file(file_obj) remove_file(file_obj.tmp_result_filename_reduced) remove_file(file_obj.tmp_result_filename) - remove_file(file_obj.tmp_filename) if out_file is not None: ok_files.append(out_file) return ok_files, cfg.get("publish_topic", None) From 798cd53b4648525a805ab586308f97c0400b1dba Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Apr 2024 10:47:17 +0200 Subject: [PATCH 2/7] Cahnge warning to info --- nwcsafpps_runner/prepare_nwp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 1953b66..53306db 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -108,7 +108,7 @@ def prepare_config(config_file_name): def remove_file(filename): """Remove a temporary file.""" if os.path.exists(filename): - LOG.warning("Removing tmp file: %s.", filename) + LOG.info("Removing tmp file: %s.", filename) os.remove(filename) From 645049caf91b18c4420e65262f97ef37c0162418 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Apr 2024 13:24:09 +0200 Subject: [PATCH 3/7] fixing publish for nwp data --- bin/run_nwp_preparation.py | 4 ++-- nwcsafpps_runner/message_utils.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index 4c60d1b..e5e09e0 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -45,14 +45,14 @@ def prepare_and_publish(pub, options, flens): """Prepare NWP files and publish.""" config_file_name = options.config_file - starttime = datetime.now(tz=timezone.utc) - timedelta(days=1) + starttime = datetime.now(tz=timezone.utc) - timedelta(days=14) ok_files, publish_topic = update_nwp(starttime, flens, config_file_name) if publish_topic is not None: for filename in ok_files: publish_msg = prepare_nwp_message(filename, publish_topic) LOG.debug("Will publish") LOG.debug("publish_msg") - publish_l1c(pub, publish_msg, publish_topic) + publish_l1c(pub, publish_msg, [publish_topic]) def _run_subscribe_publisher(pub, options, flens): diff --git a/nwcsafpps_runner/message_utils.py b/nwcsafpps_runner/message_utils.py index 157c0d7..6eeaec5 100644 --- a/nwcsafpps_runner/message_utils.py +++ b/nwcsafpps_runner/message_utils.py @@ -38,8 +38,7 @@ def prepare_nwp_message(result_file, publish_topic): to_send["uid"] = filename to_send['format'] = 'NWP grib' to_send['type'] = 'grib' - return Message('/' + publish_topic + '/', - "file", to_send).encode() + return to_send def prepare_l1c_message(result_file, mda, **kwargs): From 5cdaa9da5016fd4eb37ddd82dbd5a249339ea7e0 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Apr 2024 13:25:38 +0200 Subject: [PATCH 4/7] Fix last commit --- bin/run_nwp_preparation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_nwp_preparation.py b/bin/run_nwp_preparation.py index e5e09e0..466fd81 100644 --- a/bin/run_nwp_preparation.py +++ b/bin/run_nwp_preparation.py @@ -45,7 +45,7 @@ def prepare_and_publish(pub, options, flens): """Prepare NWP files and publish.""" config_file_name = options.config_file - starttime = datetime.now(tz=timezone.utc) - timedelta(days=14) + starttime = datetime.now(tz=timezone.utc) - timedelta(days=1) ok_files, publish_topic = update_nwp(starttime, flens, config_file_name) if publish_topic is not None: for filename in ok_files: From 5a75205a566b3715887fa6a796017f5fe8314ac8 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 16 Apr 2024 14:35:51 +0200 Subject: [PATCH 5/7] remove the tmp_file --- nwcsafpps_runner/prepare_nwp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nwcsafpps_runner/prepare_nwp.py b/nwcsafpps_runner/prepare_nwp.py index 53306db..1f63295 100644 --- a/nwcsafpps_runner/prepare_nwp.py +++ b/nwcsafpps_runner/prepare_nwp.py @@ -236,6 +236,7 @@ def update_nwp_inner(starttime, nlengths, cfg): out_file = create_nwp_file(file_obj) remove_file(file_obj.tmp_result_filename_reduced) remove_file(file_obj.tmp_result_filename) + remove_file(file_obj.tmp_filename) if out_file is not None: ok_files.append(out_file) return ok_files, cfg.get("publish_topic", None) From 1bdf7999ae2ac0ef964f658f678642b9fe2bc492 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Thu, 18 Apr 2024 12:17:41 +0200 Subject: [PATCH 6/7] fix tests --- nwcsafpps_runner/tests/test_nwp_prepare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nwcsafpps_runner/tests/test_nwp_prepare.py b/nwcsafpps_runner/tests/test_nwp_prepare.py index e4441fa..d579627 100644 --- a/nwcsafpps_runner/tests/test_nwp_prepare.py +++ b/nwcsafpps_runner/tests/test_nwp_prepare.py @@ -99,8 +99,8 @@ def test_nwp_message(self): """Test the nwp message.""" filename = "dummy_dir/PPS_ECMWF_202205100000+009H00M" publish_msg = prepare_nwp_message(filename, "dummy_topic") - expected_uri = '"uri": "dummy_dir/PPS_ECMWF_202205100000+009H00M"' - assert expected_uri in publish_msg + expected_uri = "dummy_dir/PPS_ECMWF_202205100000+009H00M" + assert publish_msg["uri"] == expected_uri class TestNWPprepareRunner: From b3f434ffeabcb074916b8e6f5bd5e31d43f95e93 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Fri, 19 Apr 2024 14:31:31 +0200 Subject: [PATCH 7/7] Check that tmp files are removed --- nwcsafpps_runner/tests/test_nwp_prepare.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nwcsafpps_runner/tests/test_nwp_prepare.py b/nwcsafpps_runner/tests/test_nwp_prepare.py index d579627..9267bde 100644 --- a/nwcsafpps_runner/tests/test_nwp_prepare.py +++ b/nwcsafpps_runner/tests/test_nwp_prepare.py @@ -22,6 +22,7 @@ # along with this program. If not, see . """Test the nwp_prepare runner code.""" +import glob import logging import os import unittest @@ -116,6 +117,8 @@ def test_update_nwp(self, fake_file_dir): # Run again when file is already created nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) assert os.path.exists(outfile) + out_files = glob.glob(os.path.join(str(my_temp_dir), "*_202205100000+009H00M*")) + assert len(out_files) == 1 def test_update_nwp_no_requirement_file(self, fake_file_dir): """Create file no requirement file.""" @@ -127,6 +130,8 @@ def test_update_nwp_no_requirement_file(self, fake_file_dir): date = datetime(year=2022, month=5, day=10, hour=0, tzinfo=timezone.utc) nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) assert os.path.exists(outfile) + out_files = glob.glob(os.path.join(str(my_temp_dir), "*_202205100000+009H00M*")) + assert len(out_files) == 1 def test_update_nwp_missing_fields(self, fake_file_dir): """Test that no file without mandatory data is created.""" @@ -136,6 +141,8 @@ def test_update_nwp_missing_fields(self, fake_file_dir): date = datetime(year=2022, month=5, day=10, hour=0, tzinfo=timezone.utc) nwc_prep.update_nwp(date - timedelta(days=2), [9], cfg_file) assert not (os.path.exists(outfile)) + out_files = glob.glob(os.path.join(str(my_temp_dir), "*_202205100000+009H00M*")) + assert len(out_files) == 0 def test_remove_filename(self, fake_file_dir): """Test the function for removing files."""