Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove/avoid redundant function calls #5280

Merged
merged 5 commits into from
Apr 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 27 additions & 37 deletions conda_build/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from conda.base.context import context
from conda.gateways.disk.read import compute_sum

from . import exceptions, utils, variants
from . import exceptions, utils
from .conda_interface import MatchSpec
from .config import Config, get_or_merge_config
from .features import feature_list
Expand All @@ -34,6 +34,15 @@
insert_variant_versions,
on_win,
)
from .variants import (
dict_of_lists_to_list_of_dicts,
find_used_variables_in_batch_script,
find_used_variables_in_shell_script,
find_used_variables_in_text,
get_default_variant,
get_vars,
list_of_dicts_to_dict_of_lists,
)

if TYPE_CHECKING:
from typing import Literal
Expand Down Expand Up @@ -156,7 +165,7 @@ def get_selectors(config: Config) -> dict[str, bool]:
if arch == "32":
d["x86"] = plat.endswith(("-32", "-64"))

defaults = variants.get_default_variant(config)
defaults = get_default_variant(config)
py = config.variant.get("python", defaults["python"])
# there are times when python comes in as a tuple
if not hasattr(py, "split"):
Expand Down Expand Up @@ -2448,9 +2457,7 @@ def append_parent_metadata(self, out_metadata):

def get_reduced_variant_set(self, used_variables):
# reduce variable space to limit work we need to do
full_collapsed_variants = variants.list_of_dicts_to_dict_of_lists(
self.config.variants
)
full_collapsed_variants = list_of_dicts_to_dict_of_lists(self.config.variants)
reduced_collapsed_variants = full_collapsed_variants.copy()
reduce_keys = set(self.config.variants[0].keys()) - set(used_variables)

Expand Down Expand Up @@ -2482,7 +2489,7 @@ def get_reduced_variant_set(self, used_variables):
# save only one element from this key
reduced_collapsed_variants[key] = utils.ensure_list(next(iter(values)))

out = variants.dict_of_lists_to_list_of_dicts(reduced_collapsed_variants)
out = dict_of_lists_to_list_of_dicts(reduced_collapsed_variants)
return out

def get_output_metadata_set(
Expand Down Expand Up @@ -2633,21 +2640,14 @@ def get_output_metadata_set(
return output_tuples

def get_loop_vars(self):
_variants = (
self.config.input_variants
if hasattr(self.config, "input_variants")
else self.config.variants
)
return variants.get_vars(_variants, loop_only=True)
return get_vars(getattr(self.config, "input_variants", self.config.variants))

def get_used_loop_vars(self, force_top_level=False, force_global=False):
return {
var
for var in self.get_used_vars(
force_top_level=force_top_level, force_global=force_global
)
if var in self.get_loop_vars()
}
loop_vars = self.get_loop_vars()
used_vars = self.get_used_vars(
force_top_level=force_top_level, force_global=force_global
)
return set(loop_vars).intersection(used_vars)

def get_rendered_recipe_text(
self, permit_undefined_jinja=False, extract_pattern=None
Expand Down Expand Up @@ -2827,7 +2827,7 @@ def _get_used_vars_meta_yaml(self, force_top_level=False, force_global=False):
apply_selectors=False,
)

all_used_selectors = variants.find_used_variables_in_text(
all_used_selectors = find_used_variables_in_text(
variant_keys, recipe_text, selectors_only=True
)

Expand All @@ -2836,7 +2836,7 @@ def _get_used_vars_meta_yaml(self, force_top_level=False, force_global=False):
force_global=force_global,
apply_selectors=True,
)
all_used_reqs = variants.find_used_variables_in_text(
all_used_reqs = find_used_variables_in_text(
variant_keys, recipe_text, selectors_only=False
)

Expand All @@ -2847,9 +2847,7 @@ def _get_used_vars_meta_yaml(self, force_top_level=False, force_global=False):
if force_global:
used = all_used
else:
requirements_used = variants.find_used_variables_in_text(
variant_keys, reqs_text
)
requirements_used = find_used_variables_in_text(variant_keys, reqs_text)
outside_reqs_used = all_used - requirements_used

requirements_used = trim_build_only_deps(self, requirements_used)
Expand All @@ -2862,16 +2860,12 @@ def _get_used_vars_build_scripts(self):
buildsh = os.path.join(self.path, "build.sh")
if os.path.isfile(buildsh):
used_vars.update(
variants.find_used_variables_in_shell_script(
self.config.variant, buildsh
)
find_used_variables_in_shell_script(self.config.variant, buildsh)
)
bldbat = os.path.join(self.path, "bld.bat")
if self.config.platform == "win" and os.path.isfile(bldbat):
used_vars.update(
variants.find_used_variables_in_batch_script(
self.config.variant, bldbat
)
find_used_variables_in_batch_script(self.config.variant, bldbat)
)
return used_vars

Expand All @@ -2884,15 +2878,11 @@ def _get_used_vars_output_script(self):
script = os.path.join(self.path, this_output["script"])
if os.path.splitext(script)[1] == ".sh":
used_vars.update(
variants.find_used_variables_in_shell_script(
self.config.variant, script
)
find_used_variables_in_shell_script(self.config.variant, script)
)
elif os.path.splitext(script)[1] == ".bat":
used_vars.update(
variants.find_used_variables_in_batch_script(
self.config.variant, script
)
find_used_variables_in_batch_script(self.config.variant, script)
)
else:
log = utils.get_logger(__name__)
Expand All @@ -2903,7 +2893,7 @@ def _get_used_vars_output_script(self):
return used_vars

def get_variants_as_dict_of_lists(self):
return variants.list_of_dicts_to_dict_of_lists(self.config.variants)
return list_of_dicts_to_dict_of_lists(self.config.variants)

def clean(self):
"""This ensures that clean is called with the correct build id"""
Expand Down
34 changes: 21 additions & 13 deletions conda_build/variants.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,28 @@
"""This file handles the parsing of feature specifications from files,
ending up with a configuration matrix"""

from __future__ import annotations

import os.path
import re
import sys
from collections import OrderedDict
from copy import copy
from functools import lru_cache
from itertools import product
from typing import TYPE_CHECKING

import yaml
from conda.base.context import context

from .conda_interface import cc_conda_build
from .deprecations import deprecated
from .utils import ensure_list, get_logger, islist, on_win, trim_empty_keys
from .version import _parse as parse_version

if TYPE_CHECKING:
from typing import Any, Iterable

DEFAULT_VARIANTS = {
"python": f"{sys.version_info.major}.{sys.version_info.minor}",
"numpy": {
Expand Down Expand Up @@ -694,21 +701,22 @@ def get_package_variants(recipedir_or_metadata, config=None, variants=None):
return filter_combined_spec_to_used_keys(combined_spec, specs=specs)


def get_vars(variants, loop_only=False):
@deprecated.argument("24.5", "24.7", "loop_only")
def get_vars(variants: Iterable[dict[str, Any]]) -> set[str]:
Comment on lines +704 to +705
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Opted to deprecate loop_only since get_vars is only invoked from one place and for that call loop_only=True (see conda_build.metadata.MetaData.get_loop_vars).

"""For purposes of naming/identifying, provide a way of identifying which variables contribute
to the matrix dimensionality"""
special_keys = {"pin_run_as_build", "zip_keys", "ignore_version"}
special_keys.update(set(ensure_list(variants[0].get("extend_keys"))))
loop_vars = [
k
for k in variants[0]
if k not in special_keys
and (
not loop_only
or any(variant[k] != variants[0][k] for variant in variants[1:])
)
]
return loop_vars
first, *others = variants
special_keys = {
"pin_run_as_build",
"zip_keys",
"ignore_version",
*ensure_list(first.get("extend_keys")),
}
return {
var
for var in set(first) - special_keys
if any(first[var] != other[var] for other in others)
}


@lru_cache(maxsize=None)
Expand Down
19 changes: 19 additions & 0 deletions news/5280-deprecate-get_vars-loop_only
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
### Enhancements

* <news item>

### Bug fixes

* <news item>

### Deprecations

* Deprecate `conda_build.variants.get_vars(loop_only)`. Unused. (#5280)

### Docs

* <news item>

### Other

* <news item>
15 changes: 15 additions & 0 deletions tests/test_variants.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
dict_of_lists_to_list_of_dicts,
filter_combined_spec_to_used_keys,
get_package_variants,
get_vars,
validate_spec,
)

Expand Down Expand Up @@ -700,3 +701,17 @@ def test_zip_key_filtering(
}

assert filter_combined_spec_to_used_keys(combined_spec, specs=specs) == expected


def test_get_vars():
variants = [
{
"python": "3.12",
"nodejs": "20",
"zip_keys": [], # ignored
},
{"python": "3.12", "nodejs": "18"},
{"python": "3.12", "nodejs": "20"},
]

assert get_vars(variants) == {"nodejs"}
Loading