Remove selectors for empty packages. (#357)

* Remove selectors for empty packages. It must be configured in the config.yaml and for pypi packages only.

grayskull/base/track_packages.py

@@ -18,6 +18,7 @@ class ConfigPkg:
     conda_forge: str = ""
     delimiter_min: str = ""
     delimiter_max: str = ""
+    avoid_selector: bool = False
 
     def __post_init__(self):
         if not self.import_name:

grayskull/strategy/config.yaml

@@ -421,3 +421,28 @@ xxhash:
 kaleido:
   conda_forge: python-kaleido
   import_name: kaleido
+
+dataclasses:
+  conda_forge: dataclasses
+  import_name: dataclasses
+  avoid_selector: true
+
+typing:
+  conda_forge: typing
+  import_name: typing
+  avoid_selector: true
+
+import_metadata:
+  conda_forge: import_metadata
+  import_name: import_metadata
+  avoid_selector: true
+
+enum34:
+  conda_forge: enum34
+  import_name: enum
+  avoid_selector: true
+
+pywin32:
+  conda_forge: pywin32
+  import_name: pywin
+  avoid_selector: true

grayskull/strategy/py_base.py

@@ -518,21 +518,13 @@ def is_compiler_present() -> bool:
         re_compiler = re.compile(
             r"^\s*[<{]\{\s*compiler\(['\"]\w+['\"]\)\s*\}\}\s*$", re.MULTILINE
         )
-        for build in requirements["build"]:
-            if re_compiler.match(build):
-                return True
-        return False
+        return any(re_compiler.match(build) for build in requirements["build"])
 
     if not is_compiler_present():
         return
 
     def clean_list_pkg(pkg, list_pkgs):
-        result = []
-        for p in list_pkgs:
-            if pkg == p.strip().split(" ", 1)[0]:
-                continue
-            result.append(p)
-        return result
+        return [p for p in list_pkgs if pkg != p.strip().split(" ", 1)[0]]
 
     for pkg in requirements["host"]:
         pkg_name = RE_DEPS_NAME.match(pkg).group(0)
@@ -753,8 +745,7 @@ def ensure_pep440(pkg: str) -> str:
     for constrain in list_constrains:
         if "~=" in constrain:
             version = constrain.strip().replace("~=", "").strip()
-            version_reduced = ".".join(version.split(".")[:-1])
-            version_reduced += ".*"
+            version_reduced = ".".join(version.split(".")[:-1]) + ".*"
             full_constrain.append(f">={version},=={version_reduced}")
         else:
             full_constrain.append(constrain.strip())

grayskull/strategy/pypi.py

@@ -13,7 +13,7 @@
 
 from grayskull.base.github import generate_git_archive_tarball_url, handle_gh_version
 from grayskull.base.pkg_info import normalize_pkg_name
-from grayskull.base.track_packages import solve_list_pkg_name
+from grayskull.base.track_packages import _get_track_info_from_file, solve_list_pkg_name
 from grayskull.cli.stdout import print_msg, print_requirements, progressbar_with_status
 from grayskull.config import Configuration
 from grayskull.strategy.abstract_strategy import AbstractStrategy
@@ -301,9 +301,7 @@ def get_run_req_from_requires_dist(requires_dist: List, config: Configuration) -
             if skip_pypi_requirement(list_extra):
                 continue
 
-            result_selector = get_all_selectors_pypi(list_extra, config)
-
-            if result_selector:
+            if result_selector := get_all_selectors_pypi(list_extra, config):
                 selector = " ".join(result_selector)
                 selector = f"  # [{selector}]"
             else:
@@ -325,8 +323,7 @@ def get_all_selectors_pypi(list_extra: List, config: Configuration) -> List:
     result_selector = []
     for extra in list_extra:
         config.is_arch = True
-        selector = parse_extra_metadata_to_selector(extra[1], extra[2], extra[3])
-        if selector:
+        if selector := parse_extra_metadata_to_selector(extra[1], extra[2], extra[3]):
             if extra[0]:
                 result_selector.append(extra[0])
             result_selector.append(selector)
@@ -408,8 +405,7 @@ def get_metadata(recipe, config) -> dict:
         get_test_requirements(metadata, config.extras_require_test)
     )
     if any("pytest" in req for req in test_requirements):
-        for module in test_imports:
-            test_commands.append("pytest --pyargs " + module)
+        test_commands.extend(f"pytest --pyargs {module}" for module in test_imports)
     test_commands.extend(get_test_entry_points(metadata.get("entry_points", [])))
     return {
         "package": {"name": name, "version": metadata["version"]},
@@ -468,8 +464,7 @@ def extract_requirements(metadata: dict, config, recipe) -> dict:
         config.is_arch = True
 
     if config.is_arch:
-        version_to_selector = py_version_to_selector(metadata, config)
-        if version_to_selector:
+        if version_to_selector := py_version_to_selector(metadata, config):
             try:
                 recipe["build"]["skip"] = True
             except (TypeError, KeyError):
@@ -484,6 +479,10 @@ def extract_requirements(metadata: dict, config, recipe) -> dict:
     if "pip" not in host_req:
         host_req += [f"python{limit_python}", "pip"]
     run_req.insert(0, f"python{limit_python}")
+
+    if config.is_strict_cf:
+        host_req = remove_selectors_pkgs_if_needed(host_req)
+        run_req = remove_selectors_pkgs_if_needed(run_req)
     result = {}
     if build_req:
         result = {
@@ -498,3 +497,17 @@ def extract_requirements(metadata: dict, config, recipe) -> dict:
     )
     update_requirements_with_pin(result)
     return result
+
+
+def remove_selectors_pkgs_if_needed(
+    list_req: List, config_file: Optional[Path] = None
+) -> List:
+    info_pkgs = _get_track_info_from_file(config_file or PYPI_CONFIG)
+    re_selector = re.compile(r"\s+#\s+\[.*", re.DOTALL)
+    result = []
+    for pkg in list_req:
+        pkg_cfg_info = info_pkgs.get(pkg.strip().split()[0], {})
+        if pkg_cfg_info.get("avoid_selector", False):
+            pkg = re_selector.sub("", pkg)
+        result.append(pkg)
+    return result

grayskull/utils.py

@@ -9,11 +9,10 @@
 from glob import glob
 from pathlib import Path
 from shutil import copyfile
-from typing import Any, List, Optional, Union
+from typing import List, Optional, Union
 
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
-from souschef.ingredient import IngredientList
 from souschef.section import Section
 
 log = logging.getLogger(__name__)
@@ -29,7 +28,12 @@ def get_std_modules() -> List:
     from stdlib_list import stdlib_list
 
     all_libs = set()
-    for py_ver in ("2.7", "3.6", "3.7", "3.8"):
+    for py_ver in (
+        "2.7",
+        "3.6",
+        "3.7",
+        "3.8",
+    ):
         all_libs.update(stdlib_list(py_ver))
     return list(all_libs)
 
@@ -45,9 +49,8 @@ def visit_Import(node):
     def visit_ImportFrom(node):
         # if node.module is missing it's a "from . import ..." statement
         # if level > 0 it's a "from .submodule import ..." statement
-        if node.module is not None and node.level == 0:
-            if node.module:
-                modules.add(node.module.split(".")[0])
+        if node.module is not None and node.level == 0 and node.module:
+            modules.add(node.module.split(".")[0])
 
     node_iter = ast.NodeVisitor()
     node_iter.visit_Import = visit_Import
@@ -157,21 +160,6 @@ def format_dependencies(all_dependencies: List, name: str) -> List:
     return formatted_dependencies
 
 
-def populate_metadata_from_dict(metadata: Any, section: Section) -> Section:
-    if not isinstance(metadata, bool) and not metadata:
-        return section
-    if isinstance(metadata, list):
-        section.value = IngredientList(section.yaml)
-        return section
-    if isinstance(metadata, dict):
-        for name, value in metadata.items():
-            if isinstance(value, bool) or value:
-                section.add_section({name: value})
-    else:
-        section.value = metadata
-    return section
-
-
 def generate_recipe(
     recipe,
     config,

tests/test_pypi.py

@@ -39,6 +39,7 @@
     get_sha256_from_pypi_metadata,
     get_url_filename,
     merge_pypi_sdist_metadata,
+    remove_selectors_pkgs_if_needed,
 )
 from grayskull.utils import PyVer, format_dependencies, generate_recipe
 
@@ -974,3 +975,33 @@ def test_no_sdist_pkg_pypi():
         AttributeError, match="There is no sdist package on pypi for arn"
     ):
         recipe, _ = create_python_recipe("arn", version="0.1.5")
+
+
+def test_remove_selectors_pkgs_if_needed():
+    assert remove_selectors_pkgs_if_needed(
+        [
+            "import_metadata >1.0  # [py>3]",
+            "pywin32  # [win]",
+            "requests >=2.0  # [unix]",
+        ]
+    ) == ["import_metadata >1.0", "pywin32", "requests >=2.0  # [unix]"]
+
+
+def test_remove_selectors_pkgs_if_needed_with_recipe():
+    recipe, _ = create_python_recipe("transformers", is_strict_cf=True, version="4.3.3")
+    assert set(recipe["requirements"]["run"]).issubset(
+        {
+            "dataclasses",
+            "filelock",
+            "importlib-metadata",
+            "numpy >=1.17",
+            "packaging",
+            "python",
+            "regex !=2019.12.17",
+            "requests",
+            "sacremoses",
+            "tokenizers <0.11,>=0.10.1",
+            "tokenizers >=0.10.1,<0.11",
+            "tqdm >=4.27",
+        }
+    )