From d8cb6f4cdacc9c02a593c9eb091f823efc7d8fe4 Mon Sep 17 00:00:00 2001 From: JSKenyon Date: Tue, 9 Apr 2024 14:36:30 +0200 Subject: [PATCH 1/6] Honour always tag even when step selection is in use. --- stimela/kitchen/recipe.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stimela/kitchen/recipe.py b/stimela/kitchen/recipe.py index 941e69cc..b5e8bdd9 100644 --- a/stimela/kitchen/recipe.py +++ b/stimela/kitchen/recipe.py @@ -385,6 +385,9 @@ def process_specifier_list(specs: List[str], num=0): if len(self.steps) != len(tagged_steps): self.log.info(f"{len(self.steps) - len(tagged_steps)} step(s) skipped due to tags ({', '.join(skip_tags)})") + always_steps = {k for k, v in self.steps.items() if "always" in v.tags} + self.log.info(f"{len(always_steps)} step(s) selected via 'always' tag: ({', '.join(always_steps)})") + # add steps explicitly enabled by --step if step_ranges: all_step_names = list(self.steps.keys()) @@ -415,7 +418,7 @@ def process_specifier_list(specs: List[str], num=0): step_subset.add(name) # specified subset becomes *the* subset self.log.info(f"{len(step_subset)} step(s) selected by name") - tagged_steps = step_subset + tagged_steps = step_subset | always_steps if not tagged_steps: self.log.info("no steps have been selected for execution") From f7f6a0efdcdc66d88779da0a626cd246f4e12927 Mon Sep 17 00:00:00 2001 From: JSKenyon Date: Tue, 9 Apr 2024 16:12:41 +0200 Subject: [PATCH 2/6] Add option to explicitly disable steps. --- stimela/commands/run.py | 6 +++++- stimela/kitchen/recipe.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/stimela/commands/run.py b/stimela/commands/run.py index 48080a8d..0f6e3df5 100644 --- a/stimela/commands/run.py +++ b/stimela/commands/run.py @@ -158,6 +158,9 @@ def load_recipe_files(filenames: List[str]): help="""only runs specific step(s) from the recipe. Use commas, or give multiple times to cherry-pick steps. Use [BEGIN]:[END] to specify a range of steps. Note that cherry-picking an individual step via this option also impies --enable-step.""") +@click.option("-k", "--skip-step", "skip_steps", metavar="STEP(s)", multiple=True, + help="""Forcefully skip the spcified step. This is option is primarily for skipping specific steps in + a tagged group.""") @click.option("-t", "--tags", "tags", metavar="TAG(s)", multiple=True, help="""only runs steps wth the given tags (and also steps tagged as "always"). Use commas, or give multiple times for multiple tags.""") @@ -195,6 +198,7 @@ def run(parameters: List[str] = [], dry_run: bool = False, last_recipe: bool = F config_equals: List[str] = [], config_assign: List[Tuple[str, str]] = [], step_ranges: List[str] = [], tags: List[str] = [], skip_tags: List[str] = [], enable_steps: List[str] = [], + skip_steps: List[str] = [], build=False, rebuild=False, build_skips=False, enable_native=False, enable_singularity=False, @@ -414,7 +418,7 @@ def log_available_runnables(): # select recipe substeps based on command line, and exit if nothing to run if not build_skips: selection_options = [] - for opts in (tags, skip_tags, step_ranges, enable_steps): + for opts in (tags, skip_tags, step_ranges, skip_steps, enable_steps): selection_options.append(set(itertools.chain(*(opt.split(",") for opt in opts)))) try: diff --git a/stimela/kitchen/recipe.py b/stimela/kitchen/recipe.py index b5e8bdd9..13311213 100644 --- a/stimela/kitchen/recipe.py +++ b/stimela/kitchen/recipe.py @@ -335,7 +335,7 @@ def enable_step(self, label, enable=True): step.skip = step._skip = True def restrict_steps(self, tags: List[str] = [], skip_tags: List[str] = [], - step_ranges: List[str] = [], enable_steps: List[str]=[]): + step_ranges: List[str] = [], skip_steps: List[str] = [], enable_steps: List[str]=[]): try: # extract subsets of tags and step specifications that refer to sub-recipes # this will map name -> (tags, skip_tags, step_ranges, enable_steps). Name is '' for the parent recipe. @@ -420,6 +420,10 @@ def process_specifier_list(specs: List[str], num=0): self.log.info(f"{len(step_subset)} step(s) selected by name") tagged_steps = step_subset | always_steps + if skip_steps: + self.log.info(f"{len(skip_steps)} step(s) explicitly skipped: ({', '.join(skip_steps)})") + tagged_steps -= skip_steps # Forcefully skipped steps apply last. + if not tagged_steps: self.log.info("no steps have been selected for execution") return 0 From 8993dcf009a4afeb0e91c5bb8172c98214495e41 Mon Sep 17 00:00:00 2001 From: JSKenyon Date: Wed, 10 Apr 2024 12:05:49 +0200 Subject: [PATCH 3/6] Checkpoint work on partial rewite of restrict_steps. --- stimela/commands/run.py | 11 ++-- stimela/kitchen/recipe.py | 122 ++++++++++++++++---------------------- stimela/kitchen/utils.py | 33 +++++++++++ 3 files changed, 90 insertions(+), 76 deletions(-) create mode 100644 stimela/kitchen/utils.py diff --git a/stimela/commands/run.py b/stimela/commands/run.py index 0f6e3df5..6cc3fc9a 100644 --- a/stimela/commands/run.py +++ b/stimela/commands/run.py @@ -158,9 +158,10 @@ def load_recipe_files(filenames: List[str]): help="""only runs specific step(s) from the recipe. Use commas, or give multiple times to cherry-pick steps. Use [BEGIN]:[END] to specify a range of steps. Note that cherry-picking an individual step via this option also impies --enable-step.""") -@click.option("-k", "--skip-step", "skip_steps", metavar="STEP(s)", multiple=True, - help="""Forcefully skip the spcified step. This is option is primarily for skipping specific steps in - a tagged group.""") +@click.option("-k", "--skip-step", "skip_ranges", metavar="STEP(s)", multiple=True, + help="""forcefully skip specific step(s) from the recipe. Use commas, or give multiple times to + cherry-pick steps. Use [BEGIN]:[END] to specify a range of steps. Note that cherry-picking an + individual step via this option also impies --enable-step.""") @click.option("-t", "--tags", "tags", metavar="TAG(s)", multiple=True, help="""only runs steps wth the given tags (and also steps tagged as "always"). Use commas, or give multiple times for multiple tags.""") @@ -198,7 +199,7 @@ def run(parameters: List[str] = [], dry_run: bool = False, last_recipe: bool = F config_equals: List[str] = [], config_assign: List[Tuple[str, str]] = [], step_ranges: List[str] = [], tags: List[str] = [], skip_tags: List[str] = [], enable_steps: List[str] = [], - skip_steps: List[str] = [], + skip_ranges: List[str] = [], build=False, rebuild=False, build_skips=False, enable_native=False, enable_singularity=False, @@ -418,7 +419,7 @@ def log_available_runnables(): # select recipe substeps based on command line, and exit if nothing to run if not build_skips: selection_options = [] - for opts in (tags, skip_tags, step_ranges, skip_steps, enable_steps): + for opts in (tags, skip_tags, step_ranges, skip_ranges, enable_steps): selection_options.append(set(itertools.chain(*(opt.split(",") for opt in opts)))) try: diff --git a/stimela/kitchen/recipe.py b/stimela/kitchen/recipe.py index 13311213..b8645b8e 100644 --- a/stimela/kitchen/recipe.py +++ b/stimela/kitchen/recipe.py @@ -25,6 +25,7 @@ from stimela import task_stats from stimela import backends from stimela.backends import StimelaBackendSchema +from stimela.kitchen.utils import keys_from_sel_string class DeferredAlias(Unresolved): @@ -334,11 +335,17 @@ def enable_step(self, label, enable=True): self.log.warning(f"will skip step '{label}'") step.skip = step._skip = True - def restrict_steps(self, tags: List[str] = [], skip_tags: List[str] = [], - step_ranges: List[str] = [], skip_steps: List[str] = [], enable_steps: List[str]=[]): + def restrict_steps( + self, + tags: List[str] = [], + skip_tags: List[str] = [], + step_ranges: List[str] = [], + skip_ranges: List[str] = [], + enable_steps: List[str] = [] + ): try: # extract subsets of tags and step specifications that refer to sub-recipes - # this will map name -> (tags, skip_tags, step_ranges, enable_steps). Name is '' for the parent recipe. + # this will map name -> (tags, skip_tags, step_ranges, enable_steps). Name is None for the parent recipe. subrecipe_entries = OrderedDict() def process_specifier_list(specs: List[str], num=0): for spec in specs: @@ -348,90 +355,63 @@ def process_specifier_list(specs: List[str], num=0): raise StepSelectionError(f"'{subrecipe}.{spec}' does not refer to a valid subrecipe") else: subrecipe = None - entry = subrecipe_entries.setdefault(subrecipe, ([],[],[],[])) + entry = subrecipe_entries.setdefault(subrecipe, ([],[],[],[],[])) entry[num].append(spec) # this builds up all the entries given on the command-line - for num, options in enumerate((tags, skip_tags, step_ranges, enable_steps)): + for num, options in enumerate((tags, skip_tags, step_ranges, skip_ranges, enable_steps)): process_specifier_list(options, num) - # process our own entries - tags, skip_tags, step_ranges, enable_steps = subrecipe_entries.get(None, ([],[],[],[])) + # process our own entries - the parent recipe has None key. + tags, skip_tags, step_ranges, skip_ranges, enable_steps = subrecipe_entries.get(None, ([],[],[],[],[])) - # apply enabled steps + # We have to handle the following functionality: + # - user specifies specific tag(s) to run + # - user specifies specific tag(s) to skip + # - user specifies step(s) to run + # - user specifies step(s) to skip + # - ensure steps tagged with always run unless explicitly skipped + # - individually specified steps to run must be force enabled + + always_steps = {k for k, v in self.steps.items() if "always" in v.tags} + never_steps = {k for k, v in self.steps.items() if "never" in v.tags} + tag_selected_steps = {k for k, v in self.steps.items() for t in tags if t in v.tags} + tag_skipped_steps = {k for k, v in self.steps.items() for t in skip_tags if t in v.tags} + selected_steps = [keys_from_sel_string(self.steps, sel_string) for sel_string in step_ranges] + skipped_steps = [keys_from_sel_string(self.steps, sel_string) for sel_string in skip_ranges] + + # Steps which are singled out are special (cherry-picked). They MUST be enabled and run. + # NOTE: Single step slices (e.g last_step:) will also trigger this behaviour and may be + # worth raising a warning over. + cherry_picked_steps = set.union(*([sel for sel in selected_steps if len(sel) == 1] or [set()])) + enable_steps.extend(list(cherry_picked_steps)) + + selected_steps = set.union(*(selected_steps or [set()])) + skipped_steps = set.union(*(skipped_steps or [set()])) + + # Build up the active steps according to option priority. + active_steps = (tag_selected_steps | selected_steps) or set(self.steps.keys()) + active_steps |= always_steps + active_steps -= tag_skipped_steps + active_steps -= never_steps + active_steps -= skipped_steps + active_steps |= cherry_picked_steps + + # Enable steps explicitly enabled by the user as well as those + # implicitly enabled by cherry-picking above. for name in enable_steps: if name in self.steps: self.enable_step(name) # config file may have skip=True, but we force-enable here else: raise StepSelectionError(f"'{name}' does not refer to a valid step") - # select subset based on tags/skip_tags, this will be a list of names - tagged_steps = set() - - # if tags are given, only use steps with (tags+{"always"}-skip_tags) - if tags: - tags = set(tags) | {"always"} - tags.difference_update(skip_tags) - for step_name, step in self.steps.items(): - if (tags & step.tags): - tagged_steps.add(step_name) - self.log.info(f"step '{step_name}' selected based on tags {tags & step.tags}") - self.log.info(f"{len(tagged_steps)} of {len(self.steps)} step(s) selected via tags ({', '.join(tags)})") - # else, use steps without any tag in (skip_tags + {"never"}) - else: - skip_tags = set(skip_tags) | {"never"} - for step_name, step in self.steps.items(): - if not (skip_tags & step.tags): - tagged_steps.add(step_name) - if len(self.steps) != len(tagged_steps): - self.log.info(f"{len(self.steps) - len(tagged_steps)} step(s) skipped due to tags ({', '.join(skip_tags)})") - - always_steps = {k for k, v in self.steps.items() if "always" in v.tags} - self.log.info(f"{len(always_steps)} step(s) selected via 'always' tag: ({', '.join(always_steps)})") - - # add steps explicitly enabled by --step - if step_ranges: - all_step_names = list(self.steps.keys()) - step_subset = set() - for name in step_ranges: - if ':' in name: - begin, end = name.split(':', 1) - if begin: - try: - first = all_step_names.index(begin) - except ValueError as exc: - raise StepSelectionError(f"no such step: '{begin}' (in '{name}')") - else: - first = 0 - if end: - try: - last = all_step_names.index(end) - except ValueError as exc: - raise StepSelectionError(f"no such step: '{end}' (in '{name}')") - else: - last = len(self.steps)-1 - step_subset.update(name for name in all_step_names[first:last+1] if name in tagged_steps) - # explicit step name: enable, and add to tagged_steps - else: - if name not in all_step_names: - raise StepSelectionError(f"no such step: '{name}'") - self.enable_step(name) # config file may have skip=True, but we force-enable here - step_subset.add(name) - # specified subset becomes *the* subset - self.log.info(f"{len(step_subset)} step(s) selected by name") - tagged_steps = step_subset | always_steps - - if skip_steps: - self.log.info(f"{len(skip_steps)} step(s) explicitly skipped: ({', '.join(skip_steps)})") - tagged_steps -= skip_steps # Forcefully skipped steps apply last. - - if not tagged_steps: + if not active_steps: self.log.info("no steps have been selected for execution") return 0 else: - if len(tagged_steps) != len(self.steps): + if len(active_steps) != len(self.steps): # apply skip flags for label, step in self.steps.items(): - if label not in tagged_steps: + if label not in active_steps: step.skip = step._skip = True # remove auto-aliases associated with skipped steps diff --git a/stimela/kitchen/utils.py b/stimela/kitchen/utils.py new file mode 100644 index 00000000..0e50f636 --- /dev/null +++ b/stimela/kitchen/utils.py @@ -0,0 +1,33 @@ +from stimela import log_exception, stimelogging +from stimela.stimelogging import log_rich_payload +from stimela.exceptions import * + + +def keys_from_sel_string(dictionary: Dict[str, str], sel_string: str): + """Select keys from a dictionary based on a slice string.""" + + keys = list(dictionary.keys()) + + if ':' in sel_string: + begin, end = sel_string.split(':', 1) + if begin: + try: + first = keys.index(begin) + except ValueError as exc: + raise StepSelectionError(f"no such step: '{begin}' (in '{sel_string}')") + else: + first = 0 + if end: + try: + last = keys.index(end) + except ValueError as exc: + raise StepSelectionError(f"no such step: '{end}' (in '{sel_string}')") + else: + last = len(keys) - 1 + selected_keys = set(keys[first: last + 1]) + else: + if sel_string not in keys: + raise StepSelectionError(f"no such step: '{sel_string}'") + selected_keys = set([sel_string]) + + return selected_keys \ No newline at end of file From daddac408b844be4e19be5b2c97088ca9663f0d4 Mon Sep 17 00:00:00 2001 From: JSKenyon Date: Wed, 10 Apr 2024 13:53:41 +0200 Subject: [PATCH 4/6] Add log messages for new code. --- stimela/kitchen/recipe.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/stimela/kitchen/recipe.py b/stimela/kitchen/recipe.py index b8645b8e..33cac777 100644 --- a/stimela/kitchen/recipe.py +++ b/stimela/kitchen/recipe.py @@ -388,6 +388,14 @@ def process_specifier_list(specs: List[str], num=0): selected_steps = set.union(*(selected_steps or [set()])) skipped_steps = set.union(*(skipped_steps or [set()])) + self.log.info(f"the following step(s) are marked as always run: ({', '.join(always_steps)})") + self.log.info(f"the following step(s) are marked as never run: ({', '.join(never_steps)})") + self.log.info(f"the following step(s) have been selected by tag: ({', '.join(tag_selected_steps)})") + self.log.info(f"the following step(s) have been skipped by tag: ({', '.join(tag_skipped_steps)})") + self.log.info(f"the following step(s) have been explicitly selected: ({', '.join(selected_steps)})") + self.log.info(f"the following step(s) have been explicitly skipped: ({', '.join(skipped_steps)})") + self.log.info(f"the following step(s) have been cherry-picked: ({', '.join(cherry_picked_steps)})") + # Build up the active steps according to option priority. active_steps = (tag_selected_steps | selected_steps) or set(self.steps.keys()) active_steps |= always_steps From d737ca06440e8d8fe6a9614b2dbe14ef1ed98472 Mon Sep 17 00:00:00 2001 From: JSKenyon Date: Wed, 10 Apr 2024 13:56:04 +0200 Subject: [PATCH 5/6] Fix helpstring. --- stimela/commands/run.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stimela/commands/run.py b/stimela/commands/run.py index 6cc3fc9a..e786f715 100644 --- a/stimela/commands/run.py +++ b/stimela/commands/run.py @@ -159,9 +159,8 @@ def load_recipe_files(filenames: List[str]): Use [BEGIN]:[END] to specify a range of steps. Note that cherry-picking an individual step via this option also impies --enable-step.""") @click.option("-k", "--skip-step", "skip_ranges", metavar="STEP(s)", multiple=True, - help="""forcefully skip specific step(s) from the recipe. Use commas, or give multiple times to - cherry-pick steps. Use [BEGIN]:[END] to specify a range of steps. Note that cherry-picking an - individual step via this option also impies --enable-step.""") + help="""forcefully skip specific recipe step(s). Use commas, or give multiple times to + cherry-pick steps. Use [BEGIN]:[END] to specify a range of steps.""") @click.option("-t", "--tags", "tags", metavar="TAG(s)", multiple=True, help="""only runs steps wth the given tags (and also steps tagged as "always"). Use commas, or give multiple times for multiple tags.""") From 44ead652bea689328420ca8d405e9d5ef660366c Mon Sep 17 00:00:00 2001 From: JSKenyon Date: Wed, 10 Apr 2024 14:05:34 +0200 Subject: [PATCH 6/6] Add newline at end of file. --- stimela/kitchen/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stimela/kitchen/utils.py b/stimela/kitchen/utils.py index 0e50f636..61b29b77 100644 --- a/stimela/kitchen/utils.py +++ b/stimela/kitchen/utils.py @@ -30,4 +30,4 @@ def keys_from_sel_string(dictionary: Dict[str, str], sel_string: str): raise StepSelectionError(f"no such step: '{sel_string}'") selected_keys = set([sel_string]) - return selected_keys \ No newline at end of file + return selected_keys