From 57d59281c3fa2dddda0a6f3b49ebd1e7d991409d Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Tue, 27 Feb 2024 08:18:05 +0100 Subject: [PATCH 1/7] Properly re.escape variant variable in find_used_variables_* Signed-off-by: Marcel Bargull --- conda_build/variants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/variants.py b/conda_build/variants.py index c5bbe9a41e..1feaa0b6ef 100644 --- a/conda_build/variants.py +++ b/conda_build/variants.py @@ -763,7 +763,7 @@ def find_used_variables_in_shell_script(variant, file_path): text = f.read() used_variables = set() for v in variant: - variant_regex = r"(^[^$]*?\$\{?\s*%s\s*[\s|\}])" % v + variant_regex = rf"(^[^$]*?\$\{{?\s*{re.escape(v)}\s*[\s|\}}])" if re.search(variant_regex, text, flags=re.MULTILINE | re.DOTALL): used_variables.add(v) return used_variables @@ -774,7 +774,7 @@ def find_used_variables_in_batch_script(variant, file_path): text = f.read() used_variables = set() for v in variant: - variant_regex = r"\%" + v + r"\%" + variant_regex = rf"\%{re.escape(v)}\%" if re.search(variant_regex, text, flags=re.MULTILINE | re.DOTALL): used_variables.add(v) return used_variables From 7a78eb8bb80f4e20cd245e04c12994d5846d904c Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Tue, 27 Feb 2024 08:18:05 +0100 Subject: [PATCH 2/7] Search static strings before costlier regex search Signed-off-by: Marcel Bargull --- conda_build/metadata.py | 4 ++-- conda_build/variants.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 01f3367d03..b4631af985 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -281,8 +281,8 @@ def select_lines(data, namespace, variants_in_place): if line.lstrip().startswith("#"): # Don't bother with comment only lines continue - m = sel_pat.match(line) - if m: + # Checking for "[" and "]" before regex matching every line is a bit faster. + if "[" in line and "]" in line and (m := sel_pat.match(line)): cond = m.group(3) try: if eval_selector(cond, namespace, variants_in_place): diff --git a/conda_build/variants.py b/conda_build/variants.py index 1feaa0b6ef..b898d05bc4 100644 --- a/conda_build/variants.py +++ b/conda_build/variants.py @@ -763,6 +763,8 @@ def find_used_variables_in_shell_script(variant, file_path): text = f.read() used_variables = set() for v in variant: + if v not in text: + continue variant_regex = rf"(^[^$]*?\$\{{?\s*{re.escape(v)}\s*[\s|\}}])" if re.search(variant_regex, text, flags=re.MULTILINE | re.DOTALL): used_variables.add(v) @@ -774,6 +776,8 @@ def find_used_variables_in_batch_script(variant, file_path): text = f.read() used_variables = set() for v in variant: + if v not in text: + continue variant_regex = rf"\%{re.escape(v)}\%" if re.search(variant_regex, text, flags=re.MULTILINE | re.DOTALL): used_variables.add(v) From e86d65ecc8a2799017ff9e96329dce7c29d56e36 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Tue, 27 Feb 2024 08:18:05 +0100 Subject: [PATCH 3/7] Remove/avoid redundant function calls Signed-off-by: Marcel Bargull --- conda_build/metadata.py | 4 ++-- conda_build/variants.py | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index b4631af985..0d747d9eb1 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -1665,7 +1665,6 @@ def build_id(self): raise RuntimeError( f"Couldn't extract raw recipe text for {self.name()} output" ) - raw_recipe_text = self.extract_package_and_build_text() raw_manual_build_string = re.search(r"\s*string:", raw_recipe_text) # user setting their own build string. Don't modify it. if manual_build_string and not ( @@ -2641,12 +2640,13 @@ def get_loop_vars(self): return variants.get_vars(_variants, loop_only=True) def get_used_loop_vars(self, force_top_level=False, force_global=False): + loop_vars = set(self.get_loop_vars()) return { var for var in self.get_used_vars( force_top_level=force_top_level, force_global=force_global ) - if var in self.get_loop_vars() + if var in loop_vars } def get_rendered_recipe_text( diff --git a/conda_build/variants.py b/conda_build/variants.py index b898d05bc4..593d44e694 100644 --- a/conda_build/variants.py +++ b/conda_build/variants.py @@ -697,16 +697,14 @@ def get_package_variants(recipedir_or_metadata, config=None, variants=None): def get_vars(variants, loop_only=False): """For purposes of naming/identifying, provide a way of identifying which variables contribute to the matrix dimensionality""" + first_variant, *other_variants = variants special_keys = {"pin_run_as_build", "zip_keys", "ignore_version"} - special_keys.update(set(ensure_list(variants[0].get("extend_keys")))) + special_keys.update(set(ensure_list(first_variant.get("extend_keys")))) loop_vars = [ k - for k in variants[0] + for k, v in first_variant.items() if k not in special_keys - and ( - not loop_only - or any(variant[k] != variants[0][k] for variant in variants[1:]) - ) + and (not loop_only or any(variant[k] != v for variant in other_variants)) ] return loop_vars From 2ea651cf21fb381dafe310928b99eee974bc923e Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Tue, 27 Feb 2024 08:18:06 +0100 Subject: [PATCH 4/7] Memoize .metadata._filter_recipe_text Signed-off-by: Marcel Bargull --- conda_build/metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 0d747d9eb1..59df328c39 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -1015,6 +1015,7 @@ def get_updated_output_dict_from_reparsed_metadata(original_dict, new_outputs): return output_d +@lru_cache(maxsize=200) def _filter_recipe_text(text, extract_pattern=None): if extract_pattern: match = re.search(extract_pattern, text, flags=re.MULTILINE | re.DOTALL) From 28b2e1c04cf5d050e90309eddff310a562823edb Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Tue, 27 Feb 2024 08:18:05 +0100 Subject: [PATCH 5/7] Remove/avoid redundant function calls Signed-off-by: Marcel Bargull --- conda_build/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 59df328c39..0109f87686 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -889,8 +889,8 @@ def get_output_dicts_from_metadata(metadata, outputs=None): outputs.append(OrderedDict(name=metadata.name())) for out in outputs: if ( - "package:" in metadata.get_recipe_text() - and out.get("name") == metadata.name() + out.get("name") == metadata.name() + and "package:" in metadata.get_recipe_text() ): combine_top_level_metadata_with_output(metadata, out) return outputs From 52ce090ea0c154dfc5f382215611bf810bbdbd3c Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Thu, 21 Mar 2024 22:57:33 +0100 Subject: [PATCH 6/7] Add news/5248-render-speedups-func-calls Signed-off-by: Marcel Bargull --- news/5248-render-speedups-func-calls | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 news/5248-render-speedups-func-calls diff --git a/news/5248-render-speedups-func-calls b/news/5248-render-speedups-func-calls new file mode 100644 index 0000000000..77863ddf02 --- /dev/null +++ b/news/5248-render-speedups-func-calls @@ -0,0 +1,19 @@ +### Enhancements + +* Speed up rendering by avoiding costly/redundant function calls. (#5248) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* From 989cb9050e7495a24e9082236f52b0e8ee2b0ab1 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Mon, 25 Mar 2024 19:48:45 +0100 Subject: [PATCH 7/7] Apply cleanup/simplifications from code review Signed-off-by: Marcel Bargull Co-authored-by: Ken Odegard Signed-off-by: Marcel Bargull --- conda_build/metadata.py | 13 +++++-------- conda_build/variants.py | 8 ++++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 0109f87686..60922b438c 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -2641,14 +2641,11 @@ def get_loop_vars(self): return variants.get_vars(_variants, loop_only=True) def get_used_loop_vars(self, force_top_level=False, force_global=False): - loop_vars = set(self.get_loop_vars()) - return { - var - for var in self.get_used_vars( - force_top_level=force_top_level, force_global=force_global - ) - if var in loop_vars - } + loop_vars = self.get_loop_vars() + used_vars = self.get_used_vars( + force_top_level=force_top_level, force_global=force_global + ) + return set(loop_vars).intersection(used_vars) def get_rendered_recipe_text( self, permit_undefined_jinja=False, extract_pattern=None diff --git a/conda_build/variants.py b/conda_build/variants.py index 593d44e694..076309fa4c 100644 --- a/conda_build/variants.py +++ b/conda_build/variants.py @@ -698,8 +698,12 @@ def get_vars(variants, loop_only=False): """For purposes of naming/identifying, provide a way of identifying which variables contribute to the matrix dimensionality""" first_variant, *other_variants = variants - special_keys = {"pin_run_as_build", "zip_keys", "ignore_version"} - special_keys.update(set(ensure_list(first_variant.get("extend_keys")))) + special_keys = { + "pin_run_as_build", + "zip_keys", + "ignore_version", + *ensure_list(first_variant.get("extend_keys")), + } loop_vars = [ k for k, v in first_variant.items()