Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update jinja and selectors handling #131

Merged
merged 7 commits into from
Nov 11, 2022
Merged

update jinja and selectors handling #131

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
43a8c40
update jinja and selectors handling
cbouss Nov 10, 2022
f6c9c56
add tests and changelog
cbouss Nov 10, 2022
ea979a5
pre-commit
cbouss Nov 10, 2022
0f7ddfd
correct changelog
cbouss Nov 11, 2022
de0ba4f
Handle YAML parsing errors
cbouss Nov 11, 2022
9aaa158
add yaml_load_failure to lint names
cbouss Nov 11, 2022
d84429d
Merge branch 'main' into jinja_and_selectors
cbouss Nov 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .flake8
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[flake8]
max-line-length = 100
max-line-length = 120
ignore = E203, W503
136 changes: 35 additions & 101 deletions anaconda_linter/recipe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import sys
import tempfile
import types
from collections import defaultdict
from contextlib import redirect_stderr, redirect_stdout
from copy import deepcopy
from io import StringIO
Expand Down Expand Up @@ -151,7 +150,8 @@ class Recipe:
JINJA_VARS = {
"cran_mirror": "https://cloud.r-project.org",
"compiler": lambda x: f"compiler_{x}",
"pin_compatible": lambda x, max_pin=None, min_pin=None: f"{x}",
"pin_compatible": lambda x, max_pin=None, min_pin=None, lower_bound=None, upper_bound=None: f"{x}",
"pin_subpackage": lambda x, max_pin=None, min_pin=None, exact=False: f"{x}",
"cdt": lambda x: x,
}

Expand All @@ -171,6 +171,8 @@ def __init__(self, recipe_dir):
# These will be filled in by load_from_string()
#: Lines of the raw recipe file
self.meta_yaml: List[str] = []
#: Selectors configuration
self.selector_dict: Dict[str, Any] = {}
# Filled in by update filter
self.version_data: Dict[str, Any] = {}
#: Original recipe before modifications (updated by load_from_string)
Expand Down Expand Up @@ -200,27 +202,40 @@ def __str__(self) -> str:
def __repr__(self) -> str:
return f'{self.__class__.__name__} "{self.recipe_dir}"'

def load_from_string(self, data, selector_dict) -> "Recipe":
def load_from_string(self, data) -> "Recipe":
"""Load and `render` recipe contents from disk"""
self.meta_yaml = self.apply_selector(data, selector_dict)
self.meta_yaml = data
if not self.meta_yaml:
raise EmptyRecipe(self)
self.render()
return self

def read_conda_build_config(
self, variant_config_files: List[str] = [], exclusive_config_files: List[str] = []
self,
selector_dict,
variant_config_files: List[str] = [],
exclusive_config_files: List[str] = [],
):
self.selector_dict = selector_dict

# List conda_build_config files for linter render.
self.conda_build_config_files = utils.find_config_files(
self.recipe_dir, variant_config_files, exclusive_config_files
)
# Cache contents of conda_build_config.yaml for conda_render.
path = Path(self.recipe_dir, "conda_build_config.yaml")
if path.is_file():
self.conda_build_config = path.read_text()
else:
self.conda_build_config = ""
# Update selector dict
for cbc in self.conda_build_config_files:
with open(cbc) as f_cbc:
try:
cbc_selectors_str = self.apply_selector(f_cbc.read(), self.selector_dict)
cbc_selectors_yml = yaml.load("\n".join(cbc_selectors_str))
if cbc_selectors_yml:
for k, v in cbc_selectors_yml.items():
if type(v) is list:
self.selector_dict[k] = v[-1]
except DuplicateKeyError as err:
line = err.problem_mark.line + 1
column = err.problem_mark.column + 1
raise DuplicateKey(self, line=line, column=column)

def read_build_scripts(self):
# Cache contents of build scripts for conda_render since conda-build
Expand Down Expand Up @@ -271,19 +286,10 @@ def from_string(
"""Create new `Recipe` object from string"""
try:
recipe = cls("")
recipe.load_from_string(recipe_text, selector_dict)
except Exception as exc:
if return_exceptions:
return exc
raise exc
try:
recipe.read_conda_build_config(variant_config_files, exclusive_config_files)
except Exception as exc:
if return_exceptions:
return exc
raise exc
try:
recipe.read_build_scripts()
recipe.read_conda_build_config(
selector_dict, variant_config_files, exclusive_config_files
)
recipe.load_from_string(recipe_text)
except Exception as exc:
if return_exceptions:
return exc
Expand All @@ -308,9 +314,10 @@ def from_file(
if recipe_fname.endswith("meta.yaml"):
recipe_fname = os.path.dirname(recipe_fname)
recipe = cls(recipe_fname)
recipe.read_conda_build_config(selector_dict, variant_config_files, exclusive_config_files)
try:
with open(os.path.join(recipe_fname, "meta.yaml")) as text:
recipe.load_from_string(text.read(), selector_dict)
recipe.load_from_string(text.read())
except FileNotFoundError:
exc = MissingMetaYaml(recipe_fname)
if return_exceptions:
Expand All @@ -320,18 +327,6 @@ def from_file(
if return_exceptions:
return exc
raise exc
try:
recipe.read_conda_build_config(variant_config_files, exclusive_config_files)
except Exception as exc:
if return_exceptions:
return exc
raise exc
try:
recipe.read_build_scripts()
except Exception as exc:
if return_exceptions:
return exc
raise exc
recipe.set_original()
return recipe

Expand All @@ -350,57 +345,6 @@ def dump(self):
"""Dump recipe content"""
return "\n".join(self.meta_yaml) + "\n"

@staticmethod
def _rewrite_selector_block(text, block_top, block_left):
if not block_left:
return None # never the whole yaml
lines = text.splitlines()
block_height = 0
variants: Dict[str, List[str]] = defaultdict(list)

for block_height, line in enumerate(lines[block_top:]):
if line.strip() and not line.startswith(" " * block_left):
break
_, _, selector = line.partition("#")
if selector:
variants[selector.strip("[] ")].append(line)
else:
for variant in variants:
variants[variant].append(line)
else:
# end of file, need to add one to block height
block_height += 1

if not block_height: # empty lines?
return None
if not variants:
return None
if any(" " in v for v in variants):
# can't handle "[py2k or osx]" style things
return None

new_lines = []
for variant in variants.values():
first = True
for line in variant:
if first:
new_lines.append("".join((" " * block_left, "- ", line)))
first = False
else:
new_lines.append("".join((" " * (block_left + 2), line)))

logger.debug(
"Replacing: lines %i - %i with %i lines:\n%s\n---\n%s",
block_top,
block_top + block_height,
len(new_lines),
"\n".join(lines[block_top : block_top + block_height]),
"\n".join(new_lines),
)

lines[block_top : block_top + block_height] = new_lines
return "\n".join(lines)

def apply_selector(self, data, selector_dict):
"""Apply selectors # [...]"""
updated_data = []
Expand All @@ -422,7 +366,8 @@ def get_template(self):
# Storing it means the recipe cannot be pickled, which in turn
# means we cannot pass it to ProcessExecutors.
try:
return utils.jinja_silent_undef.from_string("\n".join(self.meta_yaml))
meta_yaml_selectors_applied = self.apply_selector(self.meta_yaml, self.selector_dict)
return utils.jinja_silent_undef.from_string("\n".join(meta_yaml_selectors_applied))
except jinja2.exceptions.TemplateSyntaxError as exc:
raise RenderFailure(self, message=exc.message, line=exc.lineno)
except jinja2.exceptions.TemplateError as exc:
Expand Down Expand Up @@ -458,18 +403,7 @@ def render(self) -> None:
except DuplicateKeyError as err:
line = err.problem_mark.line + 1
column = err.problem_mark.column + 1
logger.debug("fixing duplicate key at %i:%i", line, column)
# We may have encountered a recipe with linux/osx variants using line selectors
yaml_text = self._rewrite_selector_block(
yaml_text, err.context_mark.line, err.context_mark.column
)
if yaml_text:
try:
self.meta = yaml.load(yaml_text)
except DuplicateKeyError:
raise DuplicateKey(self, line=line, column=column)
else:
raise DuplicateKey(self, line=line, column=column)
raise DuplicateKey(self, line=line, column=column)

if (
"package" not in self.meta
Expand Down