From bc0aa459665ac5ce8aa79132a5766fb25796b276 Mon Sep 17 00:00:00 2001 From: jo-basevi Date: Wed, 11 Oct 2023 09:01:13 +1100 Subject: [PATCH] Add sorting to general method for listing output/restart dirs - Allow regex to support restart/output dirs with counter > 999 - Add test cases for restart_history smaller/greater than restart_freq --- docs/source/config.rst | 2 +- payu/experiment.py | 19 ++++++++++--------- test/test_prune_restarts.py | 3 ++- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/source/config.rst b/docs/source/config.rst index 729e37fc..4f1483a3 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -211,7 +211,7 @@ configuration. able to parse restarts files for a datetime. ``restart_history`` - Specifies the number of latest restart files to save + Specifies how many of the most recent restart files to retain regardless of `restart_freq` *The following model-based tags are typically not configured* diff --git a/payu/experiment.py b/payu/experiment.py index 36a16fe8..15f5f8cb 100644 --- a/payu/experiment.py +++ b/payu/experiment.py @@ -198,13 +198,15 @@ def max_output_index(self, output_type="output"): raise if output_dirs and len(output_dirs): - return max([int(d.lstrip(output_type)) for d in output_dirs]) + return int(output_dirs[-1].lstrip(output_type)) def list_output_dirs(self, output_type="output"): - """Return a list of restart or output directories in archive""" - naming_pattern = re.compile(fr"^{output_type}[0-9][0-9][0-9]$") - return [d for d in os.listdir(self.archive_path) + """Return a sorted list of restart or output directories in archive""" + naming_pattern = re.compile(fr"^{output_type}[0-9][0-9][0-9]+$") + dirs = [d for d in os.listdir(self.archive_path) if naming_pattern.match(d)] + dirs.sort(key=lambda d: int(d.lstrip(output_type))) + return dirs def set_stacksize(self, stacksize): @@ -1003,9 +1005,8 @@ def get_restarts_to_prune(self, if not os.path.exists(self.archive_path): return [] - # List all and sort restart directories in archive + # List all restart directories in archive restarts = self.list_output_dirs(output_type='restart') - restarts.sort(key=lambda d: int(d.lstrip('restart'))) # TODO: Previous logic was to prune all restarts if self.repeat_run # Still need to figure out what should happen in this case @@ -1043,15 +1044,15 @@ def get_restarts_to_prune(self, restart_path = os.path.join(self.archive_path, restart) try: restart_dt = self.model.get_restart_datetime(restart_path) - except NotImplementedError as e: + except NotImplementedError: print('payu: error: Date-based restart pruning is not ' f'implemented for the {self.model.model_type} ' 'model. To use integer based restart pruning, ' 'set restart_freq to an integer value.') raise - except Exception as e: + except Exception: print('payu: error: Error parsing restart directory ', - f'{restart} for a datetime. Error: {e}') + f'{restart} for a datetime to prune restarts.') raise if (next_dt is not None and restart_dt < next_dt): diff --git a/test/test_prune_restarts.py b/test/test_prune_restarts.py index ad823b3d..253e02b1 100644 --- a/test/test_prune_restarts.py +++ b/test/test_prune_restarts.py @@ -115,7 +115,8 @@ def write_test_config(restart_freq, restart_history=None): [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), (1, 1, []), - (5, 5, [1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 18, 19]), + (5, 3, [1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 18, 19, 21]), + (5, 7, [1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17]), (5, None, [1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 18, 19]) ]) def test_force_prune_restarts(restart_freq,