From 34c78a11d1fd94a621203d17eaa83151bfdfe5e0 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 19:45:11 +0100 Subject: [PATCH 01/61] Copy over Dist and related functions from conda Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 81 +++ conda_build/_legacy_conda_imports/dist.py | 366 ++++++++++ conda_build/_legacy_conda_imports/plan.py | 643 ++++++++++++++++++ conda_build/conda_interface.py | 22 +- conda_build/inspect_pkg.py | 2 +- conda_build/utils.py | 1 - 6 files changed, 1104 insertions(+), 11 deletions(-) create mode 100644 conda_build/_legacy_conda_imports/__init__.py create mode 100644 conda_build/_legacy_conda_imports/dist.py create mode 100644 conda_build/_legacy_conda_imports/plan.py diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py new file mode 100644 index 0000000000..1dd9e96b27 --- /dev/null +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -0,0 +1,81 @@ +from conda.core.index import get_index as _get_index + +from .dist import Dist +from .plan import ( + display_actions as _display_actions, + execute_actions, + execute_plan, + install_actions, +) + + +def display_actions( + actions, index, show_channel_urls=None, specs_to_remove=(), specs_to_add=() +): + if "FETCH" in actions: + actions["FETCH"] = [index[d] for d in actions["FETCH"]] + if "LINK" in actions: + actions["LINK"] = [index[d] for d in actions["LINK"]] + if "UNLINK" in actions: + actions["UNLINK"] = [index[d] for d in actions["UNLINK"]] + index = {prec: prec for prec in index.values()} + return _display_actions( + actions, index, show_channel_urls, specs_to_remove, specs_to_add + ) + + +def get_index( + channel_urls=(), + prepend=True, + platform=None, + use_local=False, + use_cache=False, + unknown=None, + prefix=None, +): + index = _get_index( + channel_urls, prepend, platform, use_local, use_cache, unknown, prefix + ) + return {Dist(prec): prec for prec in index.values()} + + +def package_cache(): + from conda.core.package_cache_data import PackageCacheData + + class package_cache: + def __contains__(self, dist): + return bool( + PackageCacheData.first_writable().get(Dist(dist).to_package_ref(), None) + ) + + def keys(self): + return (Dist(v) for v in PackageCacheData.first_writable().values()) + + def __delitem__(self, dist): + PackageCacheData.first_writable().remove(Dist(dist).to_package_ref()) + + return package_cache() + + +def linked_data(prefix, ignore_channels=False): + """Return a dictionary of the linked packages in prefix.""" + from conda.core.prefix_data import PrefixData + + pd = PrefixData(prefix) + return { + Dist(prefix_record): prefix_record + for prefix_record in pd._prefix_records.values() + } + + +def linked(prefix, ignore_channels=False): + """Return the Dists of linked packages in prefix.""" + from conda.models.enums import PackageType + + conda_package_types = PackageType.conda_package_types() + ld = linked_data(prefix, ignore_channels=ignore_channels).items() + return { + dist + for dist, prefix_rec in ld + if prefix_rec.package_type in conda_package_types + } diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py new file mode 100644 index 0000000000..95e6d98334 --- /dev/null +++ b/conda_build/_legacy_conda_imports/dist.py @@ -0,0 +1,366 @@ +# Copyright (C) 2012 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause +"""(Legacy) Low-level implementation of a Channel.""" +import re +from logging import getLogger +from typing import NamedTuple + +from conda import CondaError +from conda.auxlib.entity import Entity, EntityType, IntegerField, StringField +from conda.base.constants import ( + CONDA_PACKAGE_EXTENSIONS, + DEFAULTS_CHANNEL_NAME, + UNKNOWN_CHANNEL, +) +from conda.base.context import context +from conda.common.compat import ensure_text_type +from conda.common.constants import NULL +from conda.common.url import has_platform, is_url, join_url +from conda.deprecations import deprecated +from conda.models.channel import Channel +from conda.models.package_info import PackageInfo +from conda.models.records import PackageRecord + +log = getLogger(__name__) + + +class DistDetails(NamedTuple): + name: str + version: str + build_string: str + build_number: str + dist_name: str + fmt: str + + +deprecated.constant( + "24.3", + "24.9", + "IndexRecord", + PackageRecord, + addendum="Use `conda.models.records.PackageRecord` instead.", +) + + +class DistType(EntityType): + def __call__(cls, *args, **kwargs): + if len(args) == 1 and not kwargs: + value = args[0] + if value in Dist._cache_: + return Dist._cache_[value] + elif isinstance(value, Dist): + dist = value + elif isinstance(value, PackageRecord): + dist = Dist.from_string( + value.fn, channel_override=value.channel.canonical_name + ) + elif hasattr(value, "dist") and isinstance(value.dist, Dist): + dist = value.dist + elif isinstance(value, PackageInfo): + dist = Dist.from_string( + value.repodata_record.fn, + channel_override=value.channel.canonical_name, + ) + elif isinstance(value, Channel): + dist = Dist.from_url(value.url()) + else: + dist = Dist.from_string(value) + Dist._cache_[value] = dist + return dist + else: + return super().__call__(*args, **kwargs) + + +def strip_extension(original_dist): + for ext in CONDA_PACKAGE_EXTENSIONS: + if original_dist.endswith(ext): + original_dist = original_dist[: -len(ext)] + return original_dist + + +def split_extension(original_dist): + stripped = strip_extension(original_dist) + return stripped, original_dist[len(stripped) :] + + +class Dist(Entity, metaclass=DistType): + _cache_ = {} + _lazy_validate = True + + channel = StringField(required=False, nullable=True, immutable=True) + + dist_name = StringField(immutable=True) + name = StringField(immutable=True) + fmt = StringField(immutable=True) + version = StringField(immutable=True) + build_string = StringField(immutable=True) + build_number = IntegerField(immutable=True) + + base_url = StringField(required=False, nullable=True, immutable=True) + platform = StringField(required=False, nullable=True, immutable=True) + + def __init__( + self, + channel, + dist_name=None, + name=None, + version=None, + build_string=None, + build_number=None, + base_url=None, + platform=None, + fmt=".tar.bz2", + ): + super().__init__( + channel=channel, + dist_name=dist_name, + name=name, + version=version, + build_string=build_string, + build_number=build_number, + base_url=base_url, + platform=platform, + fmt=fmt, + ) + + def to_package_ref(self): + return PackageRecord( + channel=self.channel, + subdir=self.platform, + name=self.name, + version=self.version, + build=self.build_string, + build_number=self.build_number, + ) + + @property + def full_name(self): + return self.__str__() + + @property + def build(self): + return self.build_string + + @property + def subdir(self): + return self.platform + + @property + def pair(self): + return self.channel or DEFAULTS_CHANNEL_NAME, self.dist_name + + @property + def quad(self): + # returns: name, version, build_string, channel + parts = self.dist_name.rsplit("-", 2) + ["", ""] + return parts[0], parts[1], parts[2], self.channel or DEFAULTS_CHANNEL_NAME + + def __str__(self): + return f"{self.channel}::{self.dist_name}" if self.channel else self.dist_name + + @property + def is_feature_package(self): + return self.dist_name.endswith("@") + + @property + def is_channel(self): + return bool(self.base_url and self.platform) + + def to_filename(self, extension=None): + if self.is_feature_package: + return self.dist_name + else: + return self.dist_name + self.fmt + + def to_matchspec(self): + return " ".join(self.quad[:3]) + + def to_match_spec(self): + from conda.match_spec import MatchSpec + + base = "=".join(self.quad[:3]) + return MatchSpec(f"{self.channel}::{base}" if self.channel else base) + + @classmethod + def from_string(cls, string, channel_override=NULL): + string = str(string) + + if is_url(string) and channel_override == NULL: + return cls.from_url(string) + + if string.endswith("@"): + return cls( + channel="@", + name=string, + version="", + build_string="", + build_number=0, + dist_name=string, + ) + + REGEX_STR = ( + r"(?:([^\s\[\]]+)::)?" # optional channel + r"([^\s\[\]]+)" # 3.x dist + r"(?:\[([a-zA-Z0-9_-]+)\])?" # with_features_depends + ) + channel, original_dist, w_f_d = re.search(REGEX_STR, string).groups() + + original_dist, fmt = split_extension(original_dist) + + if channel_override != NULL: + channel = channel_override + if not channel: + channel = UNKNOWN_CHANNEL + + # enforce dist format + dist_details = cls.parse_dist_name(original_dist) + return cls( + channel=channel, + name=dist_details.name, + version=dist_details.version, + build_string=dist_details.build_string, + build_number=dist_details.build_number, + dist_name=original_dist, + fmt=fmt, + ) + + @staticmethod + def parse_dist_name(string): + original_string = string + try: + string = ensure_text_type(string) + no_fmt_string, fmt = split_extension(string) + + # remove any directory or channel information + if "::" in no_fmt_string: + dist_name = no_fmt_string.rsplit("::", 1)[-1] + else: + dist_name = no_fmt_string.rsplit("/", 1)[-1] + + parts = dist_name.rsplit("-", 2) + + name = parts[0] + version = parts[1] + build_string = parts[2] if len(parts) >= 3 else "" + build_number_as_string = "".join( + filter( + lambda x: x.isdigit(), + (build_string.rsplit("_")[-1] if build_string else "0"), + ) + ) + build_number = int(build_number_as_string) if build_number_as_string else 0 + + return DistDetails( + name, version, build_string, build_number, dist_name, fmt + ) + + except: + raise CondaError( + "dist_name is not a valid conda package: %s" % original_string + ) + + @classmethod + def from_url(cls, url): + assert is_url(url), url + if ( + not any(url.endswith(ext) for ext in CONDA_PACKAGE_EXTENSIONS) + and "::" not in url + ): + raise CondaError("url '%s' is not a conda package" % url) + + dist_details = cls.parse_dist_name(url) + if "::" in url: + url_no_tarball = url.rsplit("::", 1)[0] + platform = context.subdir + base_url = url_no_tarball.split("::")[0] + channel = str(Channel(base_url)) + else: + url_no_tarball = url.rsplit("/", 1)[0] + platform = has_platform(url_no_tarball, context.known_subdirs) + base_url = url_no_tarball.rsplit("/", 1)[0] if platform else url_no_tarball + channel = Channel(base_url).canonical_name if platform else UNKNOWN_CHANNEL + + return cls( + channel=channel, + name=dist_details.name, + version=dist_details.version, + build_string=dist_details.build_string, + build_number=dist_details.build_number, + dist_name=dist_details.dist_name, + base_url=base_url, + platform=platform, + fmt=dist_details.fmt, + ) + + def to_url(self): + if not self.base_url: + return None + filename = self.dist_name + self.fmt + return ( + join_url(self.base_url, self.platform, filename) + if self.platform + else join_url(self.base_url, filename) + ) + + def __key__(self): + return self.channel, self.dist_name + + def __lt__(self, other): + assert isinstance(other, self.__class__) + return self.__key__() < other.__key__() + + def __gt__(self, other): + assert isinstance(other, self.__class__) + return self.__key__() > other.__key__() + + def __le__(self, other): + assert isinstance(other, self.__class__) + return self.__key__() <= other.__key__() + + def __ge__(self, other): + assert isinstance(other, self.__class__) + return self.__key__() >= other.__key__() + + def __hash__(self): + # dists compare equal regardless of fmt, but fmt is taken into account for + # object identity + return hash((self.__key__(), self.fmt)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__key__() == other.__key__() + + def __ne__(self, other): + return not self.__eq__(other) + + # ############ conda-build compatibility ################ + + def split(self, sep=None, maxsplit=-1): + assert sep == "::" + return [self.channel, self.dist_name] if self.channel else [self.dist_name] + + def rsplit(self, sep=None, maxsplit=-1): + assert sep == "-" + assert maxsplit == 2 + name = f"{self.channel}::{self.quad[0]}" if self.channel else self.quad[0] + return name, self.quad[1], self.quad[2] + + def startswith(self, match): + return self.dist_name.startswith(match) + + def __contains__(self, item): + item = strip_extension(ensure_text_type(item)) + return item in self.__str__() + + @property + def fn(self): + return self.to_filename() + + +def dist_str_to_quad(dist_str): + dist_str = strip_extension(dist_str) + if "::" in dist_str: + channel_str, dist_str = dist_str.split("::", 1) + else: + channel_str = UNKNOWN_CHANNEL + name, version, build = dist_str.rsplit("-", 2) + return name, version, build, channel_str diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py new file mode 100644 index 0000000000..48d5d57001 --- /dev/null +++ b/conda_build/_legacy_conda_imports/plan.py @@ -0,0 +1,643 @@ +# Copyright (C) 2012 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause +""" +Handle the planning of installs and their execution. + +NOTE: + conda.install uses canonical package names in its interface functions, + whereas conda.resolve uses package filenames, as those are used as index + keys. We try to keep fixes to this "impedance mismatch" local to this + module. +""" + +import sys +from collections import defaultdict +from logging import getLogger + +try: + from boltons.setutils import IndexedSet +except ImportError: # pragma: no cover + from conda._vendor.boltons.setutils import IndexedSet + +from conda.base.constants import DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL +from conda.base.context import context, stack_context_default +from conda.common.io import dashlist, env_vars, time_recorder +from conda.common.iterators import groupby_to_dict as groupby +from conda.core.index import LAST_CHANNEL_URLS, _supplement_index_with_prefix +from conda.core.link import PrefixSetup, UnlinkLinkTransaction +from conda.core.solve import diff_for_unlink_link_precs +from conda.exceptions import CondaIndexError, PackagesNotFoundError +from conda.history import History +from conda.instructions import FETCH, LINK, SYMLINK_CONDA, UNLINK +from conda.models.channel import Channel, prioritize_channels +from conda.models.enums import LinkType +from conda.models.match_spec import ChannelMatch +from conda.models.prefix_graph import PrefixGraph +from conda.models.records import PackageRecord +from conda.models.version import normalized_version +from conda.resolve import MatchSpec +from conda.utils import human_bytes + +from .dist import Dist + +log = getLogger(__name__) + +# TODO: Remove conda/plan.py. This module should be almost completely deprecated now. + + +def print_dists(dists_extras): + fmt = " %-27s|%17s" + print(fmt % ("package", "build")) + print(fmt % ("-" * 27, "-" * 17)) + for prec, extra in dists_extras: + line = fmt % (prec.name + "-" + prec.version, prec.build) + if extra: + line += extra + print(line) + + +def display_actions( + actions, index, show_channel_urls=None, specs_to_remove=(), specs_to_add=() +): + prefix = actions.get("PREFIX") + builder = ["", "## Package Plan ##\n"] + if prefix: + builder.append(" environment location: %s" % prefix) + builder.append("") + if specs_to_remove: + builder.append( + " removed specs: %s" + % dashlist(sorted(str(s) for s in specs_to_remove), indent=4) + ) + builder.append("") + if specs_to_add: + builder.append( + " added / updated specs: %s" + % dashlist(sorted(str(s) for s in specs_to_add), indent=4) + ) + builder.append("") + print("\n".join(builder)) + + if show_channel_urls is None: + show_channel_urls = context.show_channel_urls + + def channel_str(rec): + if rec.get("schannel"): + return rec["schannel"] + if rec.get("url"): + return Channel(rec["url"]).canonical_name + if rec.get("channel"): + return Channel(rec["channel"]).canonical_name + return UNKNOWN_CHANNEL + + def channel_filt(s): + if show_channel_urls is False: + return "" + if show_channel_urls is None and s == DEFAULTS_CHANNEL_NAME: + return "" + return s + + if actions.get(FETCH): + print("\nThe following packages will be downloaded:\n") + + disp_lst = [] + for prec in actions[FETCH]: + assert isinstance(prec, PackageRecord) + extra = "%15s" % human_bytes(prec["size"]) + schannel = channel_filt(prec.channel.canonical_name) + if schannel: + extra += " " + schannel + disp_lst.append((prec, extra)) + print_dists(disp_lst) + + if index and len(actions[FETCH]) > 1: + num_bytes = sum(prec["size"] for prec in actions[FETCH]) + print(" " * 4 + "-" * 60) + print(" " * 43 + "Total: %14s" % human_bytes(num_bytes)) + + # package -> [oldver-oldbuild, newver-newbuild] + packages = defaultdict(lambda: list(("", ""))) + features = defaultdict(lambda: list(("", ""))) + channels = defaultdict(lambda: list(("", ""))) + records = defaultdict(lambda: list((None, None))) + linktypes = {} + + for prec in actions.get(LINK, []): + assert isinstance(prec, PackageRecord) + pkg = prec["name"] + channels[pkg][1] = channel_str(prec) + packages[pkg][1] = prec["version"] + "-" + prec["build"] + records[pkg][1] = prec + # TODO: this is a lie; may have to give this report after + # UnlinkLinkTransaction.verify() + linktypes[pkg] = LinkType.hardlink + features[pkg][1] = ",".join(prec.get("features") or ()) + for prec in actions.get(UNLINK, []): + assert isinstance(prec, PackageRecord) + pkg = prec["name"] + channels[pkg][0] = channel_str(prec) + packages[pkg][0] = prec["version"] + "-" + prec["build"] + records[pkg][0] = prec + features[pkg][0] = ",".join(prec.get("features") or ()) + + new = {p for p in packages if not packages[p][0]} + removed = {p for p in packages if not packages[p][1]} + # New packages are actually listed in the left-hand column, + # so let's move them over there + for pkg in new: + for var in (packages, features, channels, records): + var[pkg] = var[pkg][::-1] + + updated = set() + downgraded = set() + channeled = set() + oldfmt = {} + newfmt = {} + empty = True + if packages: + empty = False + maxpkg = max(len(p) for p in packages) + 1 + maxoldver = max(len(p[0]) for p in packages.values()) + maxnewver = max(len(p[1]) for p in packages.values()) + maxoldfeatures = max(len(p[0]) for p in features.values()) + maxnewfeatures = max(len(p[1]) for p in features.values()) + maxoldchannels = max(len(channel_filt(p[0])) for p in channels.values()) + maxnewchannels = max(len(channel_filt(p[1])) for p in channels.values()) + for pkg in packages: + # That's right. I'm using old-style string formatting to generate a + # string with new-style string formatting. + oldfmt[pkg] = f"{{pkg:<{maxpkg}}} {{vers[0]:<{maxoldver}}}" + if maxoldchannels: + oldfmt[pkg] += " {channels[0]:<%s}" % maxoldchannels + if features[pkg][0]: + oldfmt[pkg] += " [{features[0]:<%s}]" % maxoldfeatures + + lt = LinkType(linktypes.get(pkg, LinkType.hardlink)) + lt = "" if lt == LinkType.hardlink else (" (%s)" % lt) + if pkg in removed or pkg in new: + oldfmt[pkg] += lt + continue + + newfmt[pkg] = "{vers[1]:<%s}" % maxnewver + if maxnewchannels: + newfmt[pkg] += " {channels[1]:<%s}" % maxnewchannels + if features[pkg][1]: + newfmt[pkg] += " [{features[1]:<%s}]" % maxnewfeatures + newfmt[pkg] += lt + + P0 = records[pkg][0] + P1 = records[pkg][1] + pri0 = P0.get("priority") + pri1 = P1.get("priority") + if pri0 is None or pri1 is None: + pri0 = pri1 = 1 + try: + if str(P1.version) == "custom": + newver = str(P0.version) != "custom" + oldver = not newver + else: + # <= here means that unchanged packages will be put in updated + N0 = normalized_version(P0.version) + N1 = normalized_version(P1.version) + newver = N0 < N1 + oldver = N0 > N1 + except TypeError: + newver = P0.version < P1.version + oldver = P0.version > P1.version + oldbld = P0.build_number > P1.build_number + newbld = P0.build_number < P1.build_number + if ( + context.channel_priority + and pri1 < pri0 + and (oldver or not newver and not newbld) + ): + channeled.add(pkg) + elif newver: + updated.add(pkg) + elif pri1 < pri0 and (oldver or not newver and oldbld): + channeled.add(pkg) + elif oldver: + downgraded.add(pkg) + elif not oldbld: + updated.add(pkg) + else: + downgraded.add(pkg) + + arrow = " --> " + lead = " " * 4 + + def format(s, pkg): + chans = [channel_filt(c) for c in channels[pkg]] + return lead + s.format( + pkg=pkg + ":", vers=packages[pkg], channels=chans, features=features[pkg] + ) + + if new: + print("\nThe following NEW packages will be INSTALLED:\n") + for pkg in sorted(new): + # New packages have been moved to the "old" column for display + print(format(oldfmt[pkg], pkg)) + + if removed: + print("\nThe following packages will be REMOVED:\n") + for pkg in sorted(removed): + print(format(oldfmt[pkg], pkg)) + + if updated: + print("\nThe following packages will be UPDATED:\n") + for pkg in sorted(updated): + print(format(oldfmt[pkg] + arrow + newfmt[pkg], pkg)) + + if channeled: + print( + "\nThe following packages will be SUPERSEDED by a higher-priority channel:\n" + ) + for pkg in sorted(channeled): + print(format(oldfmt[pkg] + arrow + newfmt[pkg], pkg)) + + if downgraded: + print("\nThe following packages will be DOWNGRADED:\n") + for pkg in sorted(downgraded): + print(format(oldfmt[pkg] + arrow + newfmt[pkg], pkg)) + + if empty and actions.get(SYMLINK_CONDA): + print("\nThe following empty environments will be CREATED:\n") + print(actions["PREFIX"]) + + print() + + +def add_unlink(actions, dist): + assert isinstance(dist, Dist) + if UNLINK not in actions: + actions[UNLINK] = [] + actions[UNLINK].append(dist) + + +# ------------------------------------------------------------------- + + +def add_defaults_to_specs(r, linked, specs, update=False, prefix=None): + return + + +def _get_best_prec_match(precs): + assert precs + for chn in context.channels: + channel_matcher = ChannelMatch(chn) + prec_matches = tuple( + prec for prec in precs if channel_matcher.match(prec.channel.name) + ) + if prec_matches: + break + else: + prec_matches = precs + log.warn("Multiple packages found:%s", dashlist(prec_matches)) + return prec_matches[0] + + +def revert_actions(prefix, revision=-1, index=None): + # TODO: If revision raise a revision error, should always go back to a safe revision + h = History(prefix) + # TODO: need a History method to get user-requested specs for revision number + # Doing a revert right now messes up user-requested spec history. + # Either need to wipe out history after ``revision``, or add the correct + # history information to the new entry about to be created. + # TODO: This is wrong!!!!!!!!!! + user_requested_specs = h.get_requested_specs_map().values() + try: + target_state = { + MatchSpec.from_dist_str(dist_str) for dist_str in h.get_state(revision) + } + except IndexError: + raise CondaIndexError("no such revision: %d" % revision) + + _supplement_index_with_prefix(index, prefix) + + not_found_in_index_specs = set() + link_precs = set() + for spec in target_state: + precs = tuple(prec for prec in index.values() if spec.match(prec)) + if not precs: + not_found_in_index_specs.add(spec) + elif len(precs) > 1: + link_precs.add(_get_best_prec_match(precs)) + else: + link_precs.add(precs[0]) + + if not_found_in_index_specs: + raise PackagesNotFoundError(not_found_in_index_specs) + + final_precs = IndexedSet(PrefixGraph(link_precs).graph) # toposort + unlink_precs, link_precs = diff_for_unlink_link_precs(prefix, final_precs) + stp = PrefixSetup(prefix, unlink_precs, link_precs, (), user_requested_specs, ()) + txn = UnlinkLinkTransaction(stp) + return txn + + +# ---------------------------- Backwards compat for conda-build -------------------------- + + +@time_recorder("execute_actions") +def execute_actions(actions, index, verbose=False): # pragma: no cover + plan = _plan_from_actions(actions, index) + execute_instructions(plan, index, verbose) + + +def _plan_from_actions(actions, index): # pragma: no cover + from conda.instructions import ACTION_CODES, PREFIX, PRINT, PROGRESS, PROGRESS_COMMANDS + + if "op_order" in actions and actions["op_order"]: + op_order = actions["op_order"] + else: + op_order = ACTION_CODES + + assert PREFIX in actions and actions[PREFIX] + prefix = actions[PREFIX] + plan = [("PREFIX", "%s" % prefix)] + + unlink_link_transaction = actions.get("UNLINKLINKTRANSACTION") + if unlink_link_transaction: + raise RuntimeError() + # progressive_fetch_extract = actions.get('PROGRESSIVEFETCHEXTRACT') + # if progressive_fetch_extract: + # plan.append((PROGRESSIVEFETCHEXTRACT, progressive_fetch_extract)) + # plan.append((UNLINKLINKTRANSACTION, unlink_link_transaction)) + # return plan + + axn = actions.get("ACTION") or None + specs = actions.get("SPECS", []) + + log.debug(f"Adding plans for operations: {op_order}") + for op in op_order: + if op not in actions: + log.trace(f"action {op} not in actions") + continue + if not actions[op]: + log.trace(f"action {op} has None value") + continue + if "_" not in op: + plan.append((PRINT, "%sing packages ..." % op.capitalize())) + elif op.startswith("RM_"): + plan.append( + (PRINT, "Pruning %s packages from the cache ..." % op[3:].lower()) + ) + if op in PROGRESS_COMMANDS: + plan.append((PROGRESS, "%d" % len(actions[op]))) + for arg in actions[op]: + log.debug(f"appending value {arg} for action {op}") + plan.append((op, arg)) + + plan = _inject_UNLINKLINKTRANSACTION(plan, index, prefix, axn, specs) + + return plan + + +def _inject_UNLINKLINKTRANSACTION(plan, index, prefix, axn, specs): # pragma: no cover + from os.path import isdir + + from conda.core.link import PrefixSetup, UnlinkLinkTransaction + from conda.core.package_cache_data import ProgressiveFetchExtract + from conda.instructions import ( + LINK, + PROGRESSIVEFETCHEXTRACT, + UNLINK, + UNLINKLINKTRANSACTION, + ) + + # this is only used for conda-build at this point + first_unlink_link_idx = next( + (q for q, p in enumerate(plan) if p[0] in (UNLINK, LINK)), -1 + ) + if first_unlink_link_idx >= 0: + grouped_instructions = groupby(lambda x: x[0], plan) + unlink_dists = tuple(Dist(d[1]) for d in grouped_instructions.get(UNLINK, ())) + link_dists = tuple(Dist(d[1]) for d in grouped_instructions.get(LINK, ())) + unlink_dists, link_dists = _handle_menuinst(unlink_dists, link_dists) + + if isdir(prefix): + unlink_precs = tuple(index[d] for d in unlink_dists) + else: + # there's nothing to unlink in an environment that doesn't exist + # this is a hack for what appears to be a logic error in conda-build + # caught in tests/test_subpackages.py::test_subpackage_recipes[python_test_dep] + unlink_precs = () + link_precs = tuple(index[d] for d in link_dists) + + pfe = ProgressiveFetchExtract(link_precs) + pfe.prepare() + + stp = PrefixSetup(prefix, unlink_precs, link_precs, (), specs, ()) + plan.insert( + first_unlink_link_idx, (UNLINKLINKTRANSACTION, UnlinkLinkTransaction(stp)) + ) + plan.insert(first_unlink_link_idx, (PROGRESSIVEFETCHEXTRACT, pfe)) + elif axn in ("INSTALL", "CREATE"): + plan.insert(0, (UNLINKLINKTRANSACTION, (prefix, (), (), (), specs))) + + return plan + + +def _handle_menuinst(unlink_dists, link_dists): # pragma: no cover + from conda.common.compat import on_win + + if not on_win: + return unlink_dists, link_dists + + # Always link/unlink menuinst first/last on windows in case a subsequent + # package tries to import it to create/remove a shortcut + + # unlink + menuinst_idx = next( + (q for q, d in enumerate(unlink_dists) if d.name == "menuinst"), None + ) + if menuinst_idx is not None: + unlink_dists = ( + *unlink_dists[:menuinst_idx], + *unlink_dists[menuinst_idx + 1 :], + *unlink_dists[menuinst_idx : menuinst_idx + 1], + ) + + # link + menuinst_idx = next( + (q for q, d in enumerate(link_dists) if d.name == "menuinst"), None + ) + if menuinst_idx is not None: + link_dists = ( + *link_dists[menuinst_idx : menuinst_idx + 1], + *link_dists[:menuinst_idx], + *link_dists[menuinst_idx + 1 :], + ) + + return unlink_dists, link_dists + + +@time_recorder("execute_plan") +def install_actions( + prefix, + index, + specs, + force=False, + only_names=None, + always_copy=False, + pinned=True, + update_deps=True, + prune=False, + channel_priority_map=None, + is_update=False, + minimal_hint=False, +): # pragma: no cover + # this is for conda-build + with env_vars( + { + "CONDA_ALLOW_NON_CHANNEL_URLS": "true", + "CONDA_SOLVER_IGNORE_TIMESTAMPS": "false", + }, + stack_callback=stack_context_default, + ): + from os.path import basename + + from conda.models.channel import Channel + + if channel_priority_map: + channel_names = IndexedSet( + Channel(url).canonical_name for url in channel_priority_map + ) + channels = IndexedSet(Channel(cn) for cn in channel_names) + subdirs = IndexedSet(basename(url) for url in channel_priority_map) + else: + # a hack for when conda-build calls this function without giving channel_priority_map + if LAST_CHANNEL_URLS: + channel_priority_map = prioritize_channels(LAST_CHANNEL_URLS) + channels = IndexedSet(Channel(url) for url in channel_priority_map) + subdirs = ( + IndexedSet( + subdir for subdir in (c.subdir for c in channels) if subdir + ) + or context.subdirs + ) + else: + channels = subdirs = None + + specs = tuple(MatchSpec(spec) for spec in specs) + + from conda.core.prefix_data import PrefixData + + PrefixData._cache_.clear() + + solver_backend = context.plugin_manager.get_cached_solver_backend() + solver = solver_backend(prefix, channels, subdirs, specs_to_add=specs) + if index: + solver._index = {prec: prec for prec in index.values()} + txn = solver.solve_for_transaction(prune=prune, ignore_pinned=not pinned) + prefix_setup = txn.prefix_setups[prefix] + actions = get_blank_actions(prefix) + actions["UNLINK"].extend(Dist(prec) for prec in prefix_setup.unlink_precs) + actions["LINK"].extend(Dist(prec) for prec in prefix_setup.link_precs) + return actions + + +def get_blank_actions(prefix): # pragma: no cover + from collections import defaultdict + + from conda.instructions import ( + CHECK_EXTRACT, + CHECK_FETCH, + EXTRACT, + FETCH, + LINK, + PREFIX, + RM_EXTRACTED, + RM_FETCHED, + SYMLINK_CONDA, + UNLINK, + ) + + actions = defaultdict(list) + actions[PREFIX] = prefix + actions["op_order"] = ( + CHECK_FETCH, + RM_FETCHED, + FETCH, + CHECK_EXTRACT, + RM_EXTRACTED, + EXTRACT, + UNLINK, + LINK, + SYMLINK_CONDA, + ) + return actions + + +@time_recorder("execute_plan") +def execute_plan(old_plan, index=None, verbose=False): # pragma: no cover + """Deprecated: This should `conda.instructions.execute_instructions` instead.""" + plan = _update_old_plan(old_plan) + execute_instructions(plan, index, verbose) + + +def execute_instructions( + plan, index=None, verbose=False, _commands=None +): # pragma: no cover + """Execute the instructions in the plan + + :param plan: A list of (instruction, arg) tuples + :param index: The meta-data index + :param verbose: verbose output + :param _commands: (For testing only) dict mapping an instruction to executable if None + then the default commands will be used + """ + from conda.base.context import context + from conda.instructions import PROGRESS_COMMANDS, commands + + if _commands is None: + _commands = commands + + log.debug("executing plan %s", plan) + + state = {"i": None, "prefix": context.root_prefix, "index": index} + + for instruction, arg in plan: + log.debug(" %s(%r)", instruction, arg) + + if state["i"] is not None and instruction in PROGRESS_COMMANDS: + state["i"] += 1 + getLogger("progress.update").info((Dist(arg).dist_name, state["i"] - 1)) + cmd = _commands[instruction] + + if callable(cmd): + cmd(state, arg) + + if ( + state["i"] is not None + and instruction in PROGRESS_COMMANDS + and state["maxval"] == state["i"] + ): + state["i"] = None + getLogger("progress.stop").info(None) + + +def _update_old_plan(old_plan): # pragma: no cover + """ + Update an old plan object to work with + `conda.instructions.execute_instructions` + """ + plan = [] + for line in old_plan: + if line.startswith("#"): + continue + if " " not in line: + from conda.exceptions import ArgumentError + + raise ArgumentError(f"The instruction {line!r} takes at least one argument") + + instruction, arg = line.split(" ", 1) + plan.append((instruction, arg)) + return plan + + +if __name__ == "__main__": + # for testing new revert_actions() only + from pprint import pprint + + pprint(dict(revert_actions(sys.prefix, int(sys.argv[1])))) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 833a4339f6..d013417a5b 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations + import configparser # noqa: F401 import os from functools import partial @@ -45,23 +46,15 @@ _toposort, # noqa: F401 add_parser_channels, add_parser_prefix, - display_actions, download, - execute_actions, - execute_plan, - get_index, # noqa: F401 handle_proxy_407, hashsum_file, human_bytes, input, - install_actions, lchmod, - linked, - linked_data, md5_file, memoized, normalized_version, - package_cache, prefix_placeholder, rm_rf, spec_from_line, @@ -75,7 +68,6 @@ win_path_to_unix, ) from conda.models.channel import get_conda_build_local_url # noqa: F401 -from conda.models.dist import Dist # noqa: F401 from conda.models.records import PackageRecord, PrefixRecord from .deprecations import deprecated @@ -182,3 +174,15 @@ def get_installed_version(prefix, pkgs): # When deactivating envs (e.g. switching from root to build/test) this env var is used, # except the PR that removed this has been reverted (for now) and Windows doesn't need it. env_path_backup_var_exists = os.environ.get("CONDA_PATH_BACKUP", None) + +from ._legacy_conda_imports import ( + Dist, + display_actions, + execute_actions, + execute_plan, + get_index, + install_actions, + linked, + linked_data, + package_cache, +) diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index e8b94978c1..eac899f464 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -16,11 +16,11 @@ from typing import Iterable, Literal from conda.core.prefix_data import PrefixData -from conda.models.dist import Dist from conda.models.records import PrefixRecord from conda.resolve import MatchSpec from conda_build.conda_interface import ( + Dist, display_actions, get_index, install_actions, diff --git a/conda_build/utils.py b/conda_build/utils.py index 9f41400990..cc7dcdf44f 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -73,7 +73,6 @@ from conda.api import PackageCacheData # noqa from conda.base.constants import KNOWN_SUBDIRS from conda.core.prefix_data import PrefixData -from conda.models.dist import Dist from conda.models.records import PrefixRecord # NOQA because it is not used in this file. From 6978746b0e43d8f6f06fac62e938931f550d2654 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 19:46:14 +0100 Subject: [PATCH 02/61] Remove unused code; consolidate imports Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 9 +- conda_build/_legacy_conda_imports/dist.py | 26 +-- conda_build/_legacy_conda_imports/plan.py | 153 +++--------------- 3 files changed, 30 insertions(+), 158 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 1dd9e96b27..baf8396722 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -1,4 +1,7 @@ from conda.core.index import get_index as _get_index +from conda.core.package_cache_data import PackageCacheData +from conda.core.prefix_data import PrefixData +from conda.models.enums import PackageType from .dist import Dist from .plan import ( @@ -40,8 +43,6 @@ def get_index( def package_cache(): - from conda.core.package_cache_data import PackageCacheData - class package_cache: def __contains__(self, dist): return bool( @@ -59,8 +60,6 @@ def __delitem__(self, dist): def linked_data(prefix, ignore_channels=False): """Return a dictionary of the linked packages in prefix.""" - from conda.core.prefix_data import PrefixData - pd = PrefixData(prefix) return { Dist(prefix_record): prefix_record @@ -70,8 +69,6 @@ def linked_data(prefix, ignore_channels=False): def linked(prefix, ignore_channels=False): """Return the Dists of linked packages in prefix.""" - from conda.models.enums import PackageType - conda_package_types = PackageType.conda_package_types() ld = linked_data(prefix, ignore_channels=ignore_channels).items() return { diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 95e6d98334..8f5b3746bf 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: BSD-3-Clause """(Legacy) Low-level implementation of a Channel.""" import re -from logging import getLogger from typing import NamedTuple from conda import CondaError @@ -16,13 +15,11 @@ from conda.common.compat import ensure_text_type from conda.common.constants import NULL from conda.common.url import has_platform, is_url, join_url -from conda.deprecations import deprecated from conda.models.channel import Channel +from conda.models.match_spec import MatchSpec from conda.models.package_info import PackageInfo from conda.models.records import PackageRecord -log = getLogger(__name__) - class DistDetails(NamedTuple): name: str @@ -33,15 +30,6 @@ class DistDetails(NamedTuple): fmt: str -deprecated.constant( - "24.3", - "24.9", - "IndexRecord", - PackageRecord, - addendum="Use `conda.models.records.PackageRecord` instead.", -) - - class DistType(EntityType): def __call__(cls, *args, **kwargs): if len(args) == 1 and not kwargs: @@ -176,8 +164,6 @@ def to_matchspec(self): return " ".join(self.quad[:3]) def to_match_spec(self): - from conda.match_spec import MatchSpec - base = "=".join(self.quad[:3]) return MatchSpec(f"{self.channel}::{base}" if self.channel else base) @@ -354,13 +340,3 @@ def __contains__(self, item): @property def fn(self): return self.to_filename() - - -def dist_str_to_quad(dist_str): - dist_str = strip_extension(dist_str) - if "::" in dist_str: - channel_str, dist_str = dist_str.split("::", 1) - else: - channel_str = UNKNOWN_CHANNEL - name, version, build = dist_str.rsplit("-", 2) - return name, version, build, channel_str diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 48d5d57001..d4be37d14c 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -10,9 +10,9 @@ module. """ -import sys from collections import defaultdict from logging import getLogger +from os.path import basename, isdir try: from boltons.setutils import IndexedSet @@ -21,18 +21,35 @@ from conda.base.constants import DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL from conda.base.context import context, stack_context_default -from conda.common.io import dashlist, env_vars, time_recorder +from conda.common.io import dashlist, env_vars from conda.common.iterators import groupby_to_dict as groupby -from conda.core.index import LAST_CHANNEL_URLS, _supplement_index_with_prefix +from conda.core.index import LAST_CHANNEL_URLS +from conda.common.compat import on_win from conda.core.link import PrefixSetup, UnlinkLinkTransaction -from conda.core.solve import diff_for_unlink_link_precs -from conda.exceptions import CondaIndexError, PackagesNotFoundError -from conda.history import History -from conda.instructions import FETCH, LINK, SYMLINK_CONDA, UNLINK +from conda.core.package_cache_data import ProgressiveFetchExtract +from conda.core.prefix_data import PrefixData +from conda.exceptions import ArgumentError +from conda.instructions import ( + ACTION_CODES, + CHECK_EXTRACT, + CHECK_FETCH, + EXTRACT, + FETCH, + LINK, + PREFIX, + PRINT, + PROGRESS, + PROGRESS_COMMANDS, + PROGRESSIVEFETCHEXTRACT, + RM_EXTRACTED, + RM_FETCHED, + SYMLINK_CONDA, + UNLINK, + UNLINKLINKTRANSACTION, + commands, +) from conda.models.channel import Channel, prioritize_channels from conda.models.enums import LinkType -from conda.models.match_spec import ChannelMatch -from conda.models.prefix_graph import PrefixGraph from conda.models.records import PackageRecord from conda.models.version import normalized_version from conda.resolve import MatchSpec @@ -267,85 +284,15 @@ def format(s, pkg): print() -def add_unlink(actions, dist): - assert isinstance(dist, Dist) - if UNLINK not in actions: - actions[UNLINK] = [] - actions[UNLINK].append(dist) - - -# ------------------------------------------------------------------- - - -def add_defaults_to_specs(r, linked, specs, update=False, prefix=None): - return - - -def _get_best_prec_match(precs): - assert precs - for chn in context.channels: - channel_matcher = ChannelMatch(chn) - prec_matches = tuple( - prec for prec in precs if channel_matcher.match(prec.channel.name) - ) - if prec_matches: - break - else: - prec_matches = precs - log.warn("Multiple packages found:%s", dashlist(prec_matches)) - return prec_matches[0] - - -def revert_actions(prefix, revision=-1, index=None): - # TODO: If revision raise a revision error, should always go back to a safe revision - h = History(prefix) - # TODO: need a History method to get user-requested specs for revision number - # Doing a revert right now messes up user-requested spec history. - # Either need to wipe out history after ``revision``, or add the correct - # history information to the new entry about to be created. - # TODO: This is wrong!!!!!!!!!! - user_requested_specs = h.get_requested_specs_map().values() - try: - target_state = { - MatchSpec.from_dist_str(dist_str) for dist_str in h.get_state(revision) - } - except IndexError: - raise CondaIndexError("no such revision: %d" % revision) - - _supplement_index_with_prefix(index, prefix) - - not_found_in_index_specs = set() - link_precs = set() - for spec in target_state: - precs = tuple(prec for prec in index.values() if spec.match(prec)) - if not precs: - not_found_in_index_specs.add(spec) - elif len(precs) > 1: - link_precs.add(_get_best_prec_match(precs)) - else: - link_precs.add(precs[0]) - - if not_found_in_index_specs: - raise PackagesNotFoundError(not_found_in_index_specs) - - final_precs = IndexedSet(PrefixGraph(link_precs).graph) # toposort - unlink_precs, link_precs = diff_for_unlink_link_precs(prefix, final_precs) - stp = PrefixSetup(prefix, unlink_precs, link_precs, (), user_requested_specs, ()) - txn = UnlinkLinkTransaction(stp) - return txn - - # ---------------------------- Backwards compat for conda-build -------------------------- -@time_recorder("execute_actions") def execute_actions(actions, index, verbose=False): # pragma: no cover plan = _plan_from_actions(actions, index) execute_instructions(plan, index, verbose) def _plan_from_actions(actions, index): # pragma: no cover - from conda.instructions import ACTION_CODES, PREFIX, PRINT, PROGRESS, PROGRESS_COMMANDS if "op_order" in actions and actions["op_order"]: op_order = actions["op_order"] @@ -394,17 +341,6 @@ def _plan_from_actions(actions, index): # pragma: no cover def _inject_UNLINKLINKTRANSACTION(plan, index, prefix, axn, specs): # pragma: no cover - from os.path import isdir - - from conda.core.link import PrefixSetup, UnlinkLinkTransaction - from conda.core.package_cache_data import ProgressiveFetchExtract - from conda.instructions import ( - LINK, - PROGRESSIVEFETCHEXTRACT, - UNLINK, - UNLINKLINKTRANSACTION, - ) - # this is only used for conda-build at this point first_unlink_link_idx = next( (q for q, p in enumerate(plan) if p[0] in (UNLINK, LINK)), -1 @@ -439,8 +375,6 @@ def _inject_UNLINKLINKTRANSACTION(plan, index, prefix, axn, specs): # pragma: n def _handle_menuinst(unlink_dists, link_dists): # pragma: no cover - from conda.common.compat import on_win - if not on_win: return unlink_dists, link_dists @@ -472,7 +406,6 @@ def _handle_menuinst(unlink_dists, link_dists): # pragma: no cover return unlink_dists, link_dists -@time_recorder("execute_plan") def install_actions( prefix, index, @@ -495,10 +428,6 @@ def install_actions( }, stack_callback=stack_context_default, ): - from os.path import basename - - from conda.models.channel import Channel - if channel_priority_map: channel_names = IndexedSet( Channel(url).canonical_name for url in channel_priority_map @@ -521,8 +450,6 @@ def install_actions( specs = tuple(MatchSpec(spec) for spec in specs) - from conda.core.prefix_data import PrefixData - PrefixData._cache_.clear() solver_backend = context.plugin_manager.get_cached_solver_backend() @@ -538,21 +465,6 @@ def install_actions( def get_blank_actions(prefix): # pragma: no cover - from collections import defaultdict - - from conda.instructions import ( - CHECK_EXTRACT, - CHECK_FETCH, - EXTRACT, - FETCH, - LINK, - PREFIX, - RM_EXTRACTED, - RM_FETCHED, - SYMLINK_CONDA, - UNLINK, - ) - actions = defaultdict(list) actions[PREFIX] = prefix actions["op_order"] = ( @@ -569,7 +481,6 @@ def get_blank_actions(prefix): # pragma: no cover return actions -@time_recorder("execute_plan") def execute_plan(old_plan, index=None, verbose=False): # pragma: no cover """Deprecated: This should `conda.instructions.execute_instructions` instead.""" plan = _update_old_plan(old_plan) @@ -587,9 +498,6 @@ def execute_instructions( :param _commands: (For testing only) dict mapping an instruction to executable if None then the default commands will be used """ - from conda.base.context import context - from conda.instructions import PROGRESS_COMMANDS, commands - if _commands is None: _commands = commands @@ -627,17 +535,8 @@ def _update_old_plan(old_plan): # pragma: no cover if line.startswith("#"): continue if " " not in line: - from conda.exceptions import ArgumentError - raise ArgumentError(f"The instruction {line!r} takes at least one argument") instruction, arg = line.split(" ", 1) plan.append((instruction, arg)) return plan - - -if __name__ == "__main__": - # for testing new revert_actions() only - from pprint import pprint - - pprint(dict(revert_actions(sys.prefix, int(sys.argv[1])))) From 08ee7d797822392add8df0b040ea0ad419e355df Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 20:29:18 +0100 Subject: [PATCH 03/61] Use action constants Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 21 +++++++++++++------ conda_build/_legacy_conda_imports/plan.py | 14 ++++++------- conda_build/conda_interface.py | 6 ++++++ conda_build/environ.py | 20 +++++++++++------- conda_build/render.py | 16 ++++++++------ 5 files changed, 50 insertions(+), 27 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index baf8396722..e994cc2a88 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -2,6 +2,15 @@ from conda.core.package_cache_data import PackageCacheData from conda.core.prefix_data import PrefixData from conda.models.enums import PackageType +from conda.instructions import ( + EXTRACT, + FETCH, + LINK, + PREFIX, + RM_EXTRACTED, + RM_FETCHED, + UNLINK, +) from .dist import Dist from .plan import ( @@ -15,12 +24,12 @@ def display_actions( actions, index, show_channel_urls=None, specs_to_remove=(), specs_to_add=() ): - if "FETCH" in actions: - actions["FETCH"] = [index[d] for d in actions["FETCH"]] - if "LINK" in actions: - actions["LINK"] = [index[d] for d in actions["LINK"]] - if "UNLINK" in actions: - actions["UNLINK"] = [index[d] for d in actions["UNLINK"]] + if FETCH in actions: + actions[FETCH] = [index[d] for d in actions[FETCH]] + if LINK in actions: + actions[LINK] = [index[d] for d in actions[LINK]] + if UNLINK in actions: + actions[UNLINK] = [index[d] for d in actions[UNLINK]] index = {prec: prec for prec in index.values()} return _display_actions( actions, index, show_channel_urls, specs_to_remove, specs_to_add diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index d4be37d14c..4bb145c7fd 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -76,7 +76,7 @@ def print_dists(dists_extras): def display_actions( actions, index, show_channel_urls=None, specs_to_remove=(), specs_to_add=() ): - prefix = actions.get("PREFIX") + prefix = actions.get(PREFIX) builder = ["", "## Package Plan ##\n"] if prefix: builder.append(" environment location: %s" % prefix) @@ -279,7 +279,7 @@ def format(s, pkg): if empty and actions.get(SYMLINK_CONDA): print("\nThe following empty environments will be CREATED:\n") - print(actions["PREFIX"]) + print(actions[PREFIX]) print() @@ -301,12 +301,12 @@ def _plan_from_actions(actions, index): # pragma: no cover assert PREFIX in actions and actions[PREFIX] prefix = actions[PREFIX] - plan = [("PREFIX", "%s" % prefix)] + plan = [(PREFIX, "%s" % prefix)] - unlink_link_transaction = actions.get("UNLINKLINKTRANSACTION") + unlink_link_transaction = actions.get(UNLINKLINKTRANSACTION) if unlink_link_transaction: raise RuntimeError() - # progressive_fetch_extract = actions.get('PROGRESSIVEFETCHEXTRACT') + # progressive_fetch_extract = actions.get(PROGRESSIVEFETCHEXTRACT) # if progressive_fetch_extract: # plan.append((PROGRESSIVEFETCHEXTRACT, progressive_fetch_extract)) # plan.append((UNLINKLINKTRANSACTION, unlink_link_transaction)) @@ -459,8 +459,8 @@ def install_actions( txn = solver.solve_for_transaction(prune=prune, ignore_pinned=not pinned) prefix_setup = txn.prefix_setups[prefix] actions = get_blank_actions(prefix) - actions["UNLINK"].extend(Dist(prec) for prec in prefix_setup.unlink_precs) - actions["LINK"].extend(Dist(prec) for prec in prefix_setup.link_precs) + actions[UNLINK].extend(Dist(prec) for prec in prefix_setup.unlink_precs) + actions[LINK].extend(Dist(prec) for prec in prefix_setup.link_precs) return actions diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index d013417a5b..040e3ab834 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -176,6 +176,12 @@ def get_installed_version(prefix, pkgs): env_path_backup_var_exists = os.environ.get("CONDA_PATH_BACKUP", None) from ._legacy_conda_imports import ( + EXTRACT, + FETCH, + LINK, + PREFIX, + RM_EXTRACTED, + RM_FETCHED, Dist, display_actions, execute_actions, diff --git a/conda_build/environ.py b/conda_build/environ.py index 9e128ad511..bdef6051bc 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -23,6 +23,10 @@ from conda_build.variants import get_default_variant from .conda_interface import ( + LINK, + PREFIX, + RM_EXTRACTED, + RM_FETCHED, CondaError, LinkError, LockError, @@ -886,8 +890,8 @@ def get_install_actions( disable_pip, ) in cached_actions and last_index_ts >= index_ts: actions = cached_actions[(specs, env, subdir, channel_urls, disable_pip)].copy() - if "PREFIX" in actions: - actions["PREFIX"] = prefix + if PREFIX in actions: + actions[PREFIX] = prefix elif specs: # this is hiding output like: # Fetching package metadata ........... @@ -969,8 +973,8 @@ def get_install_actions( if not any( re.match(r"^%s(?:$|[\s=].*)" % pkg, str(dep)) for dep in specs ): - actions["LINK"] = [ - spec for spec in actions["LINK"] if spec.name != pkg + actions[LINK] = [ + spec for spec in actions[LINK] if spec.name != pkg ] utils.trim_empty_keys(actions) cached_actions[(specs, env, subdir, channel_urls, disable_pip)] = actions.copy() @@ -1092,7 +1096,7 @@ def create_env( # Set this here and use to create environ # Setting this here is important because we use it below (symlink) prefix = config.host_prefix if host else config.build_prefix - actions["PREFIX"] = prefix + actions[PREFIX] = prefix create_env( prefix, @@ -1224,8 +1228,8 @@ def clean_pkg_cache(dist, config): locks = get_pkg_dirs_locks([config.bldpkgs_dir] + pkgs_dirs, config) with utils.try_acquire_locks(locks, timeout=config.timeout): rmplan = [ - "RM_EXTRACTED {0} local::{0}".format(dist), - "RM_FETCHED {0} local::{0}".format(dist), + f"{RM_EXTRACTED} {dist} local::{dist}", + f"{RM_FETCHED} {dist} local::{dist}", ] execute_plan(rmplan) @@ -1275,6 +1279,6 @@ def get_pinned_deps(m, section): channel_urls=tuple(m.config.channel_urls), ) runtime_deps = [ - " ".join(link.dist_name.rsplit("-", 2)) for link in actions.get("LINK", []) + " ".join(link.dist_name.rsplit("-", 2)) for link in actions.get(LINK, []) ] return runtime_deps diff --git a/conda_build/render.py b/conda_build/render.py index c0f1d8be73..74d9a06128 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -38,6 +38,10 @@ ) from .conda_interface import ( + EXTRACT, + FETCH, + LINK, + PREFIX, ProgressiveFetchExtract, TemporaryDirectory, UnsatisfiableError, @@ -90,10 +94,10 @@ def bldpkg_path(m): def actions_to_pins(actions): - if "LINK" in actions: + if LINK in actions: return [ " ".join(spec.dist_name.split()[0].rsplit("-", 2)) - for spec in actions["LINK"] + for spec in actions[LINK] ] return [] @@ -345,15 +349,15 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # this should be just downloading packages. We don't need to extract them - download_actions = { - k: v for k, v in actions.items() if k in ("FETCH", "EXTRACT", "PREFIX") + k: v for k, v in actions.items() if k in (FETCH, EXTRACT, PREFIX) } - if "FETCH" in actions or "EXTRACT" in actions: + if FETCH in actions or EXTRACT in actions: # this is to force the download execute_actions(download_actions, index, verbose=m.config.debug) pkg_files = {} - packages = actions.get("LINK", []) + packages = actions.get(LINK, []) package_subset = utils.ensure_list(package_subset) selected_packages = set() if package_subset: @@ -408,7 +412,7 @@ def get_upstream_pins(m: MetaData, actions, env): downstream dependency specs. Return these additional specs.""" env_specs = m.get_value(f"requirements/{env}", []) explicit_specs = [req.split(" ")[0] for req in env_specs] if env_specs else [] - linked_packages = actions.get("LINK", []) + linked_packages = actions.get(LINK, []) linked_packages = [pkg for pkg in linked_packages if pkg.name in explicit_specs] ignore_pkgs_list = utils.ensure_list(m.get_value("build/ignore_run_exports_from")) From 1d315892454d588d5ff73b596168a4ea3e994a83 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 20:55:48 +0100 Subject: [PATCH 04/61] Copy over plan instructions from conda Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 6 +- .../_legacy_conda_imports/instructions.py | 109 ++++++++++++++++++ conda_build/_legacy_conda_imports/plan.py | 18 +-- 3 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 conda_build/_legacy_conda_imports/instructions.py diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index e994cc2a88..8c86d4770c 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -2,7 +2,9 @@ from conda.core.package_cache_data import PackageCacheData from conda.core.prefix_data import PrefixData from conda.models.enums import PackageType -from conda.instructions import ( + +from .dist import Dist +from .instructions import ( EXTRACT, FETCH, LINK, @@ -11,8 +13,6 @@ RM_FETCHED, UNLINK, ) - -from .dist import Dist from .plan import ( display_actions as _display_actions, execute_actions, diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py new file mode 100644 index 0000000000..72742f384a --- /dev/null +++ b/conda_build/_legacy_conda_imports/instructions.py @@ -0,0 +1,109 @@ +# Copyright (C) 2012 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause +"""Define the instruction set (constants) for conda operations.""" +from logging import getLogger +from os.path import isfile, join + +from conda.core.link import UnlinkLinkTransaction +from conda.core.package_cache_data import ProgressiveFetchExtract +from conda.deprecations import deprecated +from conda.exceptions import CondaFileIOError +from conda.gateways.disk.link import islink + +log = getLogger(__name__) + +# op codes +CHECK_FETCH = "CHECK_FETCH" +FETCH = "FETCH" +CHECK_EXTRACT = "CHECK_EXTRACT" +EXTRACT = "EXTRACT" +RM_EXTRACTED = "RM_EXTRACTED" +RM_FETCHED = "RM_FETCHED" +PREFIX = "PREFIX" +PRINT = "PRINT" +PROGRESS = "PROGRESS" +SYMLINK_CONDA = "SYMLINK_CONDA" +UNLINK = "UNLINK" +LINK = "LINK" +UNLINKLINKTRANSACTION = "UNLINKLINKTRANSACTION" +PROGRESSIVEFETCHEXTRACT = "PROGRESSIVEFETCHEXTRACT" + + +PROGRESS_COMMANDS = {EXTRACT, RM_EXTRACTED} +ACTION_CODES = ( + CHECK_FETCH, + FETCH, + CHECK_EXTRACT, + EXTRACT, + UNLINK, + LINK, + SYMLINK_CONDA, + RM_EXTRACTED, + RM_FETCHED, +) + + +@deprecated("23.9", "24.3", addendum="Unused.") +def PREFIX_CMD(state, prefix): + state["prefix"] = prefix + + +def PRINT_CMD(state, arg): # pragma: no cover + if arg.startswith(("Unlinking packages", "Linking packages")): + return + getLogger("conda.stdout.verbose").info(arg) + + +def FETCH_CMD(state, package_cache_entry): + raise NotImplementedError() + + +def EXTRACT_CMD(state, arg): + raise NotImplementedError() + + +def PROGRESSIVEFETCHEXTRACT_CMD(state, progressive_fetch_extract): # pragma: no cover + assert isinstance(progressive_fetch_extract, ProgressiveFetchExtract) + progressive_fetch_extract.execute() + + +def UNLINKLINKTRANSACTION_CMD(state, arg): # pragma: no cover + unlink_link_transaction = arg + assert isinstance(unlink_link_transaction, UnlinkLinkTransaction) + unlink_link_transaction.execute() + + +def check_files_in_package(source_dir, files): + for f in files: + source_file = join(source_dir, f) + if isfile(source_file) or islink(source_file): + return True + else: + raise CondaFileIOError(source_file, "File %s does not exist in tarball" % f) + + +# Map instruction to command (a python function) +commands = { + PREFIX: PREFIX_CMD, + PRINT: PRINT_CMD, + FETCH: FETCH_CMD, + PROGRESS: lambda x, y: None, + EXTRACT: EXTRACT_CMD, + RM_EXTRACTED: lambda x, y: None, + RM_FETCHED: lambda x, y: None, + UNLINK: None, + LINK: None, + SYMLINK_CONDA: lambda x, y: None, + UNLINKLINKTRANSACTION: UNLINKLINKTRANSACTION_CMD, + PROGRESSIVEFETCHEXTRACT: PROGRESSIVEFETCHEXTRACT_CMD, +} + + +OP_ORDER = ( + RM_FETCHED, + FETCH, + RM_EXTRACTED, + EXTRACT, + UNLINK, + LINK, +) diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 4bb145c7fd..630729ffb8 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -29,7 +29,15 @@ from conda.core.package_cache_data import ProgressiveFetchExtract from conda.core.prefix_data import PrefixData from conda.exceptions import ArgumentError -from conda.instructions import ( +from conda.models.channel import Channel, prioritize_channels +from conda.models.enums import LinkType +from conda.models.records import PackageRecord +from conda.models.version import normalized_version +from conda.resolve import MatchSpec +from conda.utils import human_bytes + +from .dist import Dist +from .instructions import ( ACTION_CODES, CHECK_EXTRACT, CHECK_FETCH, @@ -48,14 +56,6 @@ UNLINKLINKTRANSACTION, commands, ) -from conda.models.channel import Channel, prioritize_channels -from conda.models.enums import LinkType -from conda.models.records import PackageRecord -from conda.models.version import normalized_version -from conda.resolve import MatchSpec -from conda.utils import human_bytes - -from .dist import Dist log = getLogger(__name__) From f89d1cbbeaf877dba5f7090ae670268f1b3ae67f Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 21:06:54 +0100 Subject: [PATCH 05/61] Remove unused code Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/instructions.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py index 72742f384a..1db849e1bd 100644 --- a/conda_build/_legacy_conda_imports/instructions.py +++ b/conda_build/_legacy_conda_imports/instructions.py @@ -2,13 +2,9 @@ # SPDX-License-Identifier: BSD-3-Clause """Define the instruction set (constants) for conda operations.""" from logging import getLogger -from os.path import isfile, join from conda.core.link import UnlinkLinkTransaction from conda.core.package_cache_data import ProgressiveFetchExtract -from conda.deprecations import deprecated -from conda.exceptions import CondaFileIOError -from conda.gateways.disk.link import islink log = getLogger(__name__) @@ -43,7 +39,6 @@ ) -@deprecated("23.9", "24.3", addendum="Unused.") def PREFIX_CMD(state, prefix): state["prefix"] = prefix @@ -73,15 +68,6 @@ def UNLINKLINKTRANSACTION_CMD(state, arg): # pragma: no cover unlink_link_transaction.execute() -def check_files_in_package(source_dir, files): - for f in files: - source_file = join(source_dir, f) - if isfile(source_file) or islink(source_file): - return True - else: - raise CondaFileIOError(source_file, "File %s does not exist in tarball" % f) - - # Map instruction to command (a python function) commands = { PREFIX: PREFIX_CMD, From 0e29611a914e6808a6d996d7993d7f17f8d6e60b Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 21:07:43 +0100 Subject: [PATCH 06/61] Make "op_order" a constant Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/instructions.py | 11 +---------- conda_build/_legacy_conda_imports/plan.py | 7 ++++--- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py index 1db849e1bd..0481502354 100644 --- a/conda_build/_legacy_conda_imports/instructions.py +++ b/conda_build/_legacy_conda_imports/instructions.py @@ -24,6 +24,7 @@ UNLINKLINKTRANSACTION = "UNLINKLINKTRANSACTION" PROGRESSIVEFETCHEXTRACT = "PROGRESSIVEFETCHEXTRACT" +OP_ORDER = "op_order" PROGRESS_COMMANDS = {EXTRACT, RM_EXTRACTED} ACTION_CODES = ( @@ -83,13 +84,3 @@ def UNLINKLINKTRANSACTION_CMD(state, arg): # pragma: no cover UNLINKLINKTRANSACTION: UNLINKLINKTRANSACTION_CMD, PROGRESSIVEFETCHEXTRACT: PROGRESSIVEFETCHEXTRACT_CMD, } - - -OP_ORDER = ( - RM_FETCHED, - FETCH, - RM_EXTRACTED, - EXTRACT, - UNLINK, - LINK, -) diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 630729ffb8..7d90b3a33b 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -44,6 +44,7 @@ EXTRACT, FETCH, LINK, + OP_ORDER, PREFIX, PRINT, PROGRESS, @@ -294,8 +295,8 @@ def execute_actions(actions, index, verbose=False): # pragma: no cover def _plan_from_actions(actions, index): # pragma: no cover - if "op_order" in actions and actions["op_order"]: - op_order = actions["op_order"] + if OP_ORDER in actions and actions[OP_ORDER]: + op_order = actions[OP_ORDER] else: op_order = ACTION_CODES @@ -467,7 +468,7 @@ def install_actions( def get_blank_actions(prefix): # pragma: no cover actions = defaultdict(list) actions[PREFIX] = prefix - actions["op_order"] = ( + actions[OP_ORDER] = ( CHECK_FETCH, RM_FETCHED, FETCH, From 9bf091231279e016713cec4d6beb694eecf85c28 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 22:08:30 +0100 Subject: [PATCH 07/61] Consolidate conda imports Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 27 +++++------ .../_legacy_conda_imports/conda_imports.py | 37 +++++++++++++++ conda_build/_legacy_conda_imports/dist.py | 27 ++++++----- .../_legacy_conda_imports/instructions.py | 3 +- conda_build/_legacy_conda_imports/plan.py | 45 ++++++++++--------- 5 files changed, 92 insertions(+), 47 deletions(-) create mode 100644 conda_build/_legacy_conda_imports/conda_imports.py diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 8c86d4770c..273a52c722 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -1,8 +1,9 @@ -from conda.core.index import get_index as _get_index -from conda.core.package_cache_data import PackageCacheData -from conda.core.prefix_data import PrefixData -from conda.models.enums import PackageType - +from .conda_imports import ( + PackageCacheData as _PackageCacheData, + PackageType as _PackageType, + PrefixData as _PrefixData, + get_index as _get_index, +) from .dist import Dist from .instructions import ( EXTRACT, @@ -11,7 +12,7 @@ PREFIX, RM_EXTRACTED, RM_FETCHED, - UNLINK, + UNLINK as _UNLINK, ) from .plan import ( display_actions as _display_actions, @@ -28,8 +29,8 @@ def display_actions( actions[FETCH] = [index[d] for d in actions[FETCH]] if LINK in actions: actions[LINK] = [index[d] for d in actions[LINK]] - if UNLINK in actions: - actions[UNLINK] = [index[d] for d in actions[UNLINK]] + if _UNLINK in actions: + actions[_UNLINK] = [index[d] for d in actions[_UNLINK]] index = {prec: prec for prec in index.values()} return _display_actions( actions, index, show_channel_urls, specs_to_remove, specs_to_add @@ -55,21 +56,21 @@ def package_cache(): class package_cache: def __contains__(self, dist): return bool( - PackageCacheData.first_writable().get(Dist(dist).to_package_ref(), None) + _PackageCacheData.first_writable().get(Dist(dist).to_package_ref(), None) ) def keys(self): - return (Dist(v) for v in PackageCacheData.first_writable().values()) + return (Dist(v) for v in _PackageCacheData.first_writable().values()) def __delitem__(self, dist): - PackageCacheData.first_writable().remove(Dist(dist).to_package_ref()) + _PackageCacheData.first_writable().remove(Dist(dist).to_package_ref()) return package_cache() def linked_data(prefix, ignore_channels=False): """Return a dictionary of the linked packages in prefix.""" - pd = PrefixData(prefix) + pd = _PrefixData(prefix) return { Dist(prefix_record): prefix_record for prefix_record in pd._prefix_records.values() @@ -78,7 +79,7 @@ def linked_data(prefix, ignore_channels=False): def linked(prefix, ignore_channels=False): """Return the Dists of linked packages in prefix.""" - conda_package_types = PackageType.conda_package_types() + conda_package_types = _PackageType.conda_package_types() ld = linked_data(prefix, ignore_channels=ignore_channels).items() return { dist diff --git a/conda_build/_legacy_conda_imports/conda_imports.py b/conda_build/_legacy_conda_imports/conda_imports.py new file mode 100644 index 0000000000..b1af2eccb3 --- /dev/null +++ b/conda_build/_legacy_conda_imports/conda_imports.py @@ -0,0 +1,37 @@ +try: + from boltons.setutils import IndexedSet +except ImportError: # pragma: no cover + from conda._vendor.boltons.setutils import IndexedSet + +from conda.exports import ( + Channel, + CondaError, + MatchSpec, + PackageRecord, + ProgressiveFetchExtract, + context, + human_bytes, + on_win, + normalized_version, +) + +from conda.auxlib.entity import Entity, EntityType, IntegerField, StringField +from conda.base.constants import ( + CONDA_PACKAGE_EXTENSIONS, + DEFAULTS_CHANNEL_NAME, + UNKNOWN_CHANNEL, +) +from conda.base.context import stack_context_default +from conda.common.compat import ensure_text_type +from conda.common.constants import NULL +from conda.common.io import dashlist, env_vars +from conda.common.iterators import groupby_to_dict +from conda.common.url import has_platform, is_url, join_url +from conda.core.index import LAST_CHANNEL_URLS, get_index +from conda.core.link import PrefixSetup, UnlinkLinkTransaction +from conda.core.package_cache_data import PackageCacheData +from conda.core.prefix_data import PrefixData +from conda.exceptions import ArgumentError +from conda.models.channel import prioritize_channels +from conda.models.enums import LinkType, PackageType +from conda.models.package_info import PackageInfo diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 8f5b3746bf..6507049ae1 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -4,21 +4,26 @@ import re from typing import NamedTuple -from conda import CondaError -from conda.auxlib.entity import Entity, EntityType, IntegerField, StringField -from conda.base.constants import ( +from .conda_imports import ( CONDA_PACKAGE_EXTENSIONS, DEFAULTS_CHANNEL_NAME, + NULL, UNKNOWN_CHANNEL, + CondaError, + Channel, + Entity, + EntityType, + IntegerField, + MatchSpec, + PackageInfo, + PackageRecord, + StringField, + context, + ensure_text_type, + has_platform, + is_url, + join_url, ) -from conda.base.context import context -from conda.common.compat import ensure_text_type -from conda.common.constants import NULL -from conda.common.url import has_platform, is_url, join_url -from conda.models.channel import Channel -from conda.models.match_spec import MatchSpec -from conda.models.package_info import PackageInfo -from conda.models.records import PackageRecord class DistDetails(NamedTuple): diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py index 0481502354..d92c406ba9 100644 --- a/conda_build/_legacy_conda_imports/instructions.py +++ b/conda_build/_legacy_conda_imports/instructions.py @@ -3,8 +3,7 @@ """Define the instruction set (constants) for conda operations.""" from logging import getLogger -from conda.core.link import UnlinkLinkTransaction -from conda.core.package_cache_data import ProgressiveFetchExtract +from .conda_imports import ProgressiveFetchExtract, UnlinkLinkTransaction log = getLogger(__name__) diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 7d90b3a33b..88f17ac15f 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -14,27 +14,30 @@ from logging import getLogger from os.path import basename, isdir -try: - from boltons.setutils import IndexedSet -except ImportError: # pragma: no cover - from conda._vendor.boltons.setutils import IndexedSet - -from conda.base.constants import DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL -from conda.base.context import context, stack_context_default -from conda.common.io import dashlist, env_vars -from conda.common.iterators import groupby_to_dict as groupby -from conda.core.index import LAST_CHANNEL_URLS -from conda.common.compat import on_win -from conda.core.link import PrefixSetup, UnlinkLinkTransaction -from conda.core.package_cache_data import ProgressiveFetchExtract -from conda.core.prefix_data import PrefixData -from conda.exceptions import ArgumentError -from conda.models.channel import Channel, prioritize_channels -from conda.models.enums import LinkType -from conda.models.records import PackageRecord -from conda.models.version import normalized_version -from conda.resolve import MatchSpec -from conda.utils import human_bytes +from .conda_imports import ( + DEFAULTS_CHANNEL_NAME, + LAST_CHANNEL_URLS, + UNKNOWN_CHANNEL, + ArgumentError, + Channel, + IndexedSet, + LinkType, + MatchSpec, + PackageRecord, + PrefixData, + PrefixSetup, + ProgressiveFetchExtract, + UnlinkLinkTransaction, + context, + dashlist, + env_vars, + groupby_to_dict as groupby, + human_bytes, + normalized_version, + on_win, + prioritize_channels, + stack_context_default, +) from .dist import Dist from .instructions import ( From 7251887f7dafbb48ac75e258371aa7b36da19bd6 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 23:19:34 +0100 Subject: [PATCH 08/61] Dead code removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 7 -- .../_legacy_conda_imports/conda_imports.py | 1 - .../_legacy_conda_imports/instructions.py | 30 ----- conda_build/_legacy_conda_imports/plan.py | 113 +----------------- conda_build/conda_interface.py | 5 - conda_build/environ.py | 16 +-- conda_build/render.py | 19 +-- 7 files changed, 23 insertions(+), 168 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 273a52c722..f20337c5bd 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -6,18 +6,13 @@ ) from .dist import Dist from .instructions import ( - EXTRACT, - FETCH, LINK, PREFIX, - RM_EXTRACTED, - RM_FETCHED, UNLINK as _UNLINK, ) from .plan import ( display_actions as _display_actions, execute_actions, - execute_plan, install_actions, ) @@ -25,8 +20,6 @@ def display_actions( actions, index, show_channel_urls=None, specs_to_remove=(), specs_to_add=() ): - if FETCH in actions: - actions[FETCH] = [index[d] for d in actions[FETCH]] if LINK in actions: actions[LINK] = [index[d] for d in actions[LINK]] if _UNLINK in actions: diff --git a/conda_build/_legacy_conda_imports/conda_imports.py b/conda_build/_legacy_conda_imports/conda_imports.py index b1af2eccb3..2991edcb0c 100644 --- a/conda_build/_legacy_conda_imports/conda_imports.py +++ b/conda_build/_legacy_conda_imports/conda_imports.py @@ -10,7 +10,6 @@ PackageRecord, ProgressiveFetchExtract, context, - human_bytes, on_win, normalized_version, ) diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py index d92c406ba9..a1113638dd 100644 --- a/conda_build/_legacy_conda_imports/instructions.py +++ b/conda_build/_legacy_conda_imports/instructions.py @@ -8,16 +8,8 @@ log = getLogger(__name__) # op codes -CHECK_FETCH = "CHECK_FETCH" -FETCH = "FETCH" -CHECK_EXTRACT = "CHECK_EXTRACT" -EXTRACT = "EXTRACT" -RM_EXTRACTED = "RM_EXTRACTED" -RM_FETCHED = "RM_FETCHED" PREFIX = "PREFIX" PRINT = "PRINT" -PROGRESS = "PROGRESS" -SYMLINK_CONDA = "SYMLINK_CONDA" UNLINK = "UNLINK" LINK = "LINK" UNLINKLINKTRANSACTION = "UNLINKLINKTRANSACTION" @@ -25,17 +17,9 @@ OP_ORDER = "op_order" -PROGRESS_COMMANDS = {EXTRACT, RM_EXTRACTED} ACTION_CODES = ( - CHECK_FETCH, - FETCH, - CHECK_EXTRACT, - EXTRACT, UNLINK, LINK, - SYMLINK_CONDA, - RM_EXTRACTED, - RM_FETCHED, ) @@ -49,14 +33,6 @@ def PRINT_CMD(state, arg): # pragma: no cover getLogger("conda.stdout.verbose").info(arg) -def FETCH_CMD(state, package_cache_entry): - raise NotImplementedError() - - -def EXTRACT_CMD(state, arg): - raise NotImplementedError() - - def PROGRESSIVEFETCHEXTRACT_CMD(state, progressive_fetch_extract): # pragma: no cover assert isinstance(progressive_fetch_extract, ProgressiveFetchExtract) progressive_fetch_extract.execute() @@ -72,14 +48,8 @@ def UNLINKLINKTRANSACTION_CMD(state, arg): # pragma: no cover commands = { PREFIX: PREFIX_CMD, PRINT: PRINT_CMD, - FETCH: FETCH_CMD, - PROGRESS: lambda x, y: None, - EXTRACT: EXTRACT_CMD, - RM_EXTRACTED: lambda x, y: None, - RM_FETCHED: lambda x, y: None, UNLINK: None, LINK: None, - SYMLINK_CONDA: lambda x, y: None, UNLINKLINKTRANSACTION: UNLINKLINKTRANSACTION_CMD, PROGRESSIVEFETCHEXTRACT: PROGRESSIVEFETCHEXTRACT_CMD, } diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 88f17ac15f..3dd5b5e21c 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -18,7 +18,6 @@ DEFAULTS_CHANNEL_NAME, LAST_CHANNEL_URLS, UNKNOWN_CHANNEL, - ArgumentError, Channel, IndexedSet, LinkType, @@ -32,7 +31,6 @@ dashlist, env_vars, groupby_to_dict as groupby, - human_bytes, normalized_version, on_win, prioritize_channels, @@ -42,20 +40,11 @@ from .dist import Dist from .instructions import ( ACTION_CODES, - CHECK_EXTRACT, - CHECK_FETCH, - EXTRACT, - FETCH, LINK, OP_ORDER, PREFIX, PRINT, - PROGRESS, - PROGRESS_COMMANDS, PROGRESSIVEFETCHEXTRACT, - RM_EXTRACTED, - RM_FETCHED, - SYMLINK_CONDA, UNLINK, UNLINKLINKTRANSACTION, commands, @@ -66,17 +55,6 @@ # TODO: Remove conda/plan.py. This module should be almost completely deprecated now. -def print_dists(dists_extras): - fmt = " %-27s|%17s" - print(fmt % ("package", "build")) - print(fmt % ("-" * 27, "-" * 17)) - for prec, extra in dists_extras: - line = fmt % (prec.name + "-" + prec.version, prec.build) - if extra: - line += extra - print(line) - - def display_actions( actions, index, show_channel_urls=None, specs_to_remove=(), specs_to_add=() ): @@ -118,24 +96,6 @@ def channel_filt(s): return "" return s - if actions.get(FETCH): - print("\nThe following packages will be downloaded:\n") - - disp_lst = [] - for prec in actions[FETCH]: - assert isinstance(prec, PackageRecord) - extra = "%15s" % human_bytes(prec["size"]) - schannel = channel_filt(prec.channel.canonical_name) - if schannel: - extra += " " + schannel - disp_lst.append((prec, extra)) - print_dists(disp_lst) - - if index and len(actions[FETCH]) > 1: - num_bytes = sum(prec["size"] for prec in actions[FETCH]) - print(" " * 4 + "-" * 60) - print(" " * 43 + "Total: %14s" % human_bytes(num_bytes)) - # package -> [oldver-oldbuild, newver-newbuild] packages = defaultdict(lambda: list(("", ""))) features = defaultdict(lambda: list(("", ""))) @@ -281,10 +241,6 @@ def format(s, pkg): for pkg in sorted(downgraded): print(format(oldfmt[pkg] + arrow + newfmt[pkg], pkg)) - if empty and actions.get(SYMLINK_CONDA): - print("\nThe following empty environments will be CREATED:\n") - print(actions[PREFIX]) - print() @@ -316,9 +272,6 @@ def _plan_from_actions(actions, index): # pragma: no cover # plan.append((UNLINKLINKTRANSACTION, unlink_link_transaction)) # return plan - axn = actions.get("ACTION") or None - specs = actions.get("SPECS", []) - log.debug(f"Adding plans for operations: {op_order}") for op in op_order: if op not in actions: @@ -329,22 +282,16 @@ def _plan_from_actions(actions, index): # pragma: no cover continue if "_" not in op: plan.append((PRINT, "%sing packages ..." % op.capitalize())) - elif op.startswith("RM_"): - plan.append( - (PRINT, "Pruning %s packages from the cache ..." % op[3:].lower()) - ) - if op in PROGRESS_COMMANDS: - plan.append((PROGRESS, "%d" % len(actions[op]))) for arg in actions[op]: log.debug(f"appending value {arg} for action {op}") plan.append((op, arg)) - plan = _inject_UNLINKLINKTRANSACTION(plan, index, prefix, axn, specs) + plan = _inject_UNLINKLINKTRANSACTION(plan, index, prefix) return plan -def _inject_UNLINKLINKTRANSACTION(plan, index, prefix, axn, specs): # pragma: no cover +def _inject_UNLINKLINKTRANSACTION(plan, index, prefix): # pragma: no cover # this is only used for conda-build at this point first_unlink_link_idx = next( (q for q, p in enumerate(plan) if p[0] in (UNLINK, LINK)), -1 @@ -367,13 +314,11 @@ def _inject_UNLINKLINKTRANSACTION(plan, index, prefix, axn, specs): # pragma: n pfe = ProgressiveFetchExtract(link_precs) pfe.prepare() - stp = PrefixSetup(prefix, unlink_precs, link_precs, (), specs, ()) + stp = PrefixSetup(prefix, unlink_precs, link_precs, (), [], ()) plan.insert( first_unlink_link_idx, (UNLINKLINKTRANSACTION, UnlinkLinkTransaction(stp)) ) plan.insert(first_unlink_link_idx, (PROGRESSIVEFETCHEXTRACT, pfe)) - elif axn in ("INSTALL", "CREATE"): - plan.insert(0, (UNLINKLINKTRANSACTION, (prefix, (), (), (), specs))) return plan @@ -472,39 +417,19 @@ def get_blank_actions(prefix): # pragma: no cover actions = defaultdict(list) actions[PREFIX] = prefix actions[OP_ORDER] = ( - CHECK_FETCH, - RM_FETCHED, - FETCH, - CHECK_EXTRACT, - RM_EXTRACTED, - EXTRACT, UNLINK, LINK, - SYMLINK_CONDA, ) return actions -def execute_plan(old_plan, index=None, verbose=False): # pragma: no cover - """Deprecated: This should `conda.instructions.execute_instructions` instead.""" - plan = _update_old_plan(old_plan) - execute_instructions(plan, index, verbose) - - -def execute_instructions( - plan, index=None, verbose=False, _commands=None -): # pragma: no cover +def execute_instructions(plan, index=None, verbose=False): """Execute the instructions in the plan :param plan: A list of (instruction, arg) tuples :param index: The meta-data index :param verbose: verbose output - :param _commands: (For testing only) dict mapping an instruction to executable if None - then the default commands will be used """ - if _commands is None: - _commands = commands - log.debug("executing plan %s", plan) state = {"i": None, "prefix": context.root_prefix, "index": index} @@ -512,35 +437,7 @@ def execute_instructions( for instruction, arg in plan: log.debug(" %s(%r)", instruction, arg) - if state["i"] is not None and instruction in PROGRESS_COMMANDS: - state["i"] += 1 - getLogger("progress.update").info((Dist(arg).dist_name, state["i"] - 1)) - cmd = _commands[instruction] + cmd = commands[instruction] if callable(cmd): cmd(state, arg) - - if ( - state["i"] is not None - and instruction in PROGRESS_COMMANDS - and state["maxval"] == state["i"] - ): - state["i"] = None - getLogger("progress.stop").info(None) - - -def _update_old_plan(old_plan): # pragma: no cover - """ - Update an old plan object to work with - `conda.instructions.execute_instructions` - """ - plan = [] - for line in old_plan: - if line.startswith("#"): - continue - if " " not in line: - raise ArgumentError(f"The instruction {line!r} takes at least one argument") - - instruction, arg = line.split(" ", 1) - plan.append((instruction, arg)) - return plan diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 040e3ab834..a32215c44b 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -176,16 +176,11 @@ def get_installed_version(prefix, pkgs): env_path_backup_var_exists = os.environ.get("CONDA_PATH_BACKUP", None) from ._legacy_conda_imports import ( - EXTRACT, - FETCH, LINK, PREFIX, - RM_EXTRACTED, - RM_FETCHED, Dist, display_actions, execute_actions, - execute_plan, get_index, install_actions, linked, diff --git a/conda_build/environ.py b/conda_build/environ.py index bdef6051bc..0a2aa0ef21 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -25,8 +25,6 @@ from .conda_interface import ( LINK, PREFIX, - RM_EXTRACTED, - RM_FETCHED, CondaError, LinkError, LockError, @@ -37,7 +35,6 @@ create_default_packages, display_actions, execute_actions, - execute_plan, get_version_from_git_tag, install_actions, package_cache, @@ -1227,11 +1224,14 @@ def clean_pkg_cache(dist, config): with utils.LoggingContext(conda_log_level): locks = get_pkg_dirs_locks([config.bldpkgs_dir] + pkgs_dirs, config) with utils.try_acquire_locks(locks, timeout=config.timeout): - rmplan = [ - f"{RM_EXTRACTED} {dist} local::{dist}", - f"{RM_FETCHED} {dist} local::{dist}", - ] - execute_plan(rmplan) + # NOTE: The following out-commented execute_plan was defunct + # (RM_* were no-ops). + # + # rmplan = [ + # f"{RM_EXTRACTED} {dist} local::{dist}", + # f"{RM_FETCHED} {dist} local::{dist}", + # ] + # execute_plan(rmplan) # Conda does not seem to do a complete cleanup sometimes. This is supplemental. # Conda's cleanup is still necessary - it keeps track of its own in-memory diff --git a/conda_build/render.py b/conda_build/render.py index 74d9a06128..693c1632c5 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -38,10 +38,7 @@ ) from .conda_interface import ( - EXTRACT, - FETCH, LINK, - PREFIX, ProgressiveFetchExtract, TemporaryDirectory, UnsatisfiableError, @@ -348,12 +345,16 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # this should be just downloading packages. We don't need to extract them - - download_actions = { - k: v for k, v in actions.items() if k in (FETCH, EXTRACT, PREFIX) - } - if FETCH in actions or EXTRACT in actions: - # this is to force the download - execute_actions(download_actions, index, verbose=m.config.debug) + # NOTE: The following out-commented execute_actions was defunct + # (FETCH/EXTRACT were replaced by PROGRESSIVEFETCHEXTRACT). + # + # download_actions = { + # k: v for k, v in actions.items() if k in (FETCH, EXTRACT, PREFIX) + # } + # if FETCH in actions or EXTRACT in actions: + # # this is to force the download + # execute_actions(download_actions, index, verbose=m.config.debug) + pkg_files = {} From cb9c8b0e8239fc5c12d25ee662ba97bd339143f3 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 23:28:26 +0100 Subject: [PATCH 09/61] Dead code removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 6 +-- .../_legacy_conda_imports/instructions.py | 5 +-- conda_build/_legacy_conda_imports/plan.py | 45 ++++--------------- 3 files changed, 10 insertions(+), 46 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index f20337c5bd..4f41a81498 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -8,7 +8,6 @@ from .instructions import ( LINK, PREFIX, - UNLINK as _UNLINK, ) from .plan import ( display_actions as _display_actions, @@ -22,11 +21,8 @@ def display_actions( ): if LINK in actions: actions[LINK] = [index[d] for d in actions[LINK]] - if _UNLINK in actions: - actions[_UNLINK] = [index[d] for d in actions[_UNLINK]] - index = {prec: prec for prec in index.values()} return _display_actions( - actions, index, show_channel_urls, specs_to_remove, specs_to_add + actions, show_channel_urls, specs_to_remove, specs_to_add ) diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py index a1113638dd..691a289b32 100644 --- a/conda_build/_legacy_conda_imports/instructions.py +++ b/conda_build/_legacy_conda_imports/instructions.py @@ -10,7 +10,6 @@ # op codes PREFIX = "PREFIX" PRINT = "PRINT" -UNLINK = "UNLINK" LINK = "LINK" UNLINKLINKTRANSACTION = "UNLINKLINKTRANSACTION" PROGRESSIVEFETCHEXTRACT = "PROGRESSIVEFETCHEXTRACT" @@ -18,7 +17,6 @@ OP_ORDER = "op_order" ACTION_CODES = ( - UNLINK, LINK, ) @@ -28,7 +26,7 @@ def PREFIX_CMD(state, prefix): def PRINT_CMD(state, arg): # pragma: no cover - if arg.startswith(("Unlinking packages", "Linking packages")): + if arg.startswith(("Linking packages",)): return getLogger("conda.stdout.verbose").info(arg) @@ -48,7 +46,6 @@ def UNLINKLINKTRANSACTION_CMD(state, arg): # pragma: no cover commands = { PREFIX: PREFIX_CMD, PRINT: PRINT_CMD, - UNLINK: None, LINK: None, UNLINKLINKTRANSACTION: UNLINKLINKTRANSACTION_CMD, PROGRESSIVEFETCHEXTRACT: PROGRESSIVEFETCHEXTRACT_CMD, diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 3dd5b5e21c..df39c60cec 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -45,7 +45,6 @@ PREFIX, PRINT, PROGRESSIVEFETCHEXTRACT, - UNLINK, UNLINKLINKTRANSACTION, commands, ) @@ -56,7 +55,7 @@ def display_actions( - actions, index, show_channel_urls=None, specs_to_remove=(), specs_to_add=() + actions, show_channel_urls=None, specs_to_remove=(), specs_to_add=() ): prefix = actions.get(PREFIX) builder = ["", "## Package Plan ##\n"] @@ -113,13 +112,6 @@ def channel_filt(s): # UnlinkLinkTransaction.verify() linktypes[pkg] = LinkType.hardlink features[pkg][1] = ",".join(prec.get("features") or ()) - for prec in actions.get(UNLINK, []): - assert isinstance(prec, PackageRecord) - pkg = prec["name"] - channels[pkg][0] = channel_str(prec) - packages[pkg][0] = prec["version"] + "-" + prec["build"] - records[pkg][0] = prec - features[pkg][0] = ",".join(prec.get("features") or ()) new = {p for p in packages if not packages[p][0]} removed = {p for p in packages if not packages[p][1]} @@ -294,27 +286,19 @@ def _plan_from_actions(actions, index): # pragma: no cover def _inject_UNLINKLINKTRANSACTION(plan, index, prefix): # pragma: no cover # this is only used for conda-build at this point first_unlink_link_idx = next( - (q for q, p in enumerate(plan) if p[0] in (UNLINK, LINK)), -1 + (q for q, p in enumerate(plan) if p[0] in (LINK,)), -1 ) if first_unlink_link_idx >= 0: grouped_instructions = groupby(lambda x: x[0], plan) - unlink_dists = tuple(Dist(d[1]) for d in grouped_instructions.get(UNLINK, ())) link_dists = tuple(Dist(d[1]) for d in grouped_instructions.get(LINK, ())) - unlink_dists, link_dists = _handle_menuinst(unlink_dists, link_dists) + link_dists = _handle_menuinst(link_dists) - if isdir(prefix): - unlink_precs = tuple(index[d] for d in unlink_dists) - else: - # there's nothing to unlink in an environment that doesn't exist - # this is a hack for what appears to be a logic error in conda-build - # caught in tests/test_subpackages.py::test_subpackage_recipes[python_test_dep] - unlink_precs = () link_precs = tuple(index[d] for d in link_dists) pfe = ProgressiveFetchExtract(link_precs) pfe.prepare() - stp = PrefixSetup(prefix, unlink_precs, link_precs, (), [], ()) + stp = PrefixSetup(prefix, (), link_precs, (), [], ()) plan.insert( first_unlink_link_idx, (UNLINKLINKTRANSACTION, UnlinkLinkTransaction(stp)) ) @@ -323,24 +307,13 @@ def _inject_UNLINKLINKTRANSACTION(plan, index, prefix): # pragma: no cover return plan -def _handle_menuinst(unlink_dists, link_dists): # pragma: no cover +def _handle_menuinst(link_dists): # pragma: no cover if not on_win: - return unlink_dists, link_dists + return link_dists - # Always link/unlink menuinst first/last on windows in case a subsequent + # Always link menuinst first/last on windows in case a subsequent # package tries to import it to create/remove a shortcut - # unlink - menuinst_idx = next( - (q for q, d in enumerate(unlink_dists) if d.name == "menuinst"), None - ) - if menuinst_idx is not None: - unlink_dists = ( - *unlink_dists[:menuinst_idx], - *unlink_dists[menuinst_idx + 1 :], - *unlink_dists[menuinst_idx : menuinst_idx + 1], - ) - # link menuinst_idx = next( (q for q, d in enumerate(link_dists) if d.name == "menuinst"), None @@ -352,7 +325,7 @@ def _handle_menuinst(unlink_dists, link_dists): # pragma: no cover *link_dists[menuinst_idx + 1 :], ) - return unlink_dists, link_dists + return link_dists def install_actions( @@ -408,7 +381,6 @@ def install_actions( txn = solver.solve_for_transaction(prune=prune, ignore_pinned=not pinned) prefix_setup = txn.prefix_setups[prefix] actions = get_blank_actions(prefix) - actions[UNLINK].extend(Dist(prec) for prec in prefix_setup.unlink_precs) actions[LINK].extend(Dist(prec) for prec in prefix_setup.link_precs) return actions @@ -417,7 +389,6 @@ def get_blank_actions(prefix): # pragma: no cover actions = defaultdict(list) actions[PREFIX] = prefix actions[OP_ORDER] = ( - UNLINK, LINK, ) return actions From fca694fb11e8f9bade6e0caf343dce56e978daca Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sat, 11 Nov 2023 23:57:08 +0100 Subject: [PATCH 10/61] Move package_cache to its solve consumer Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 17 ----------------- .../_legacy_conda_imports/conda_imports.py | 1 - conda_build/conda_interface.py | 2 +- conda_build/environ.py | 13 ++++++++++++- conda_build/utils.py | 1 - 5 files changed, 13 insertions(+), 21 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 4f41a81498..77be9b90e9 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -1,5 +1,4 @@ from .conda_imports import ( - PackageCacheData as _PackageCacheData, PackageType as _PackageType, PrefixData as _PrefixData, get_index as _get_index, @@ -41,22 +40,6 @@ def get_index( return {Dist(prec): prec for prec in index.values()} -def package_cache(): - class package_cache: - def __contains__(self, dist): - return bool( - _PackageCacheData.first_writable().get(Dist(dist).to_package_ref(), None) - ) - - def keys(self): - return (Dist(v) for v in _PackageCacheData.first_writable().values()) - - def __delitem__(self, dist): - _PackageCacheData.first_writable().remove(Dist(dist).to_package_ref()) - - return package_cache() - - def linked_data(prefix, ignore_channels=False): """Return a dictionary of the linked packages in prefix.""" pd = _PrefixData(prefix) diff --git a/conda_build/_legacy_conda_imports/conda_imports.py b/conda_build/_legacy_conda_imports/conda_imports.py index 2991edcb0c..988f8089ea 100644 --- a/conda_build/_legacy_conda_imports/conda_imports.py +++ b/conda_build/_legacy_conda_imports/conda_imports.py @@ -28,7 +28,6 @@ from conda.common.url import has_platform, is_url, join_url from conda.core.index import LAST_CHANNEL_URLS, get_index from conda.core.link import PrefixSetup, UnlinkLinkTransaction -from conda.core.package_cache_data import PackageCacheData from conda.core.prefix_data import PrefixData from conda.exceptions import ArgumentError from conda.models.channel import prioritize_channels diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index a32215c44b..04f492f8c5 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -17,6 +17,7 @@ from conda.base.context import context, determine_target_prefix, reset_context from conda.base.context import non_x86_machines as non_x86_linux_machines # noqa: F401 from conda.core.package_cache import ProgressiveFetchExtract # noqa: F401 +from conda.core.package_cache_data import PackageCacheData # noqa: F401 from conda.exceptions import ( # noqa: F401 CondaError, CondaHTTPError, @@ -185,5 +186,4 @@ def get_installed_version(prefix, pkgs): install_actions, linked, linked_data, - package_cache, ) diff --git a/conda_build/environ.py b/conda_build/environ.py index 0a2aa0ef21..90fd1b3b7f 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -29,6 +29,7 @@ LinkError, LockError, NoPackagesFoundError, + PackageCacheData, PaddingError, TemporaryDirectory, UnsatisfiableError, @@ -37,7 +38,6 @@ execute_actions, get_version_from_git_tag, install_actions, - package_cache, pkgs_dirs, reset_context, root_dir, @@ -1236,6 +1236,17 @@ def clean_pkg_cache(dist, config): # Conda does not seem to do a complete cleanup sometimes. This is supplemental. # Conda's cleanup is still necessary - it keeps track of its own in-memory # list of downloaded things. + class package_cache: + def __contains__(self, dist): + return bool( + PackageCacheData.first_writable().get(Dist(dist).to_package_ref(), None) + ) + + def keys(self): + return (Dist(v) for v in PackageCacheData.first_writable().values()) + + def __delitem__(self, dist): + PackageCacheData.first_writable().remove(Dist(dist).to_package_ref()) for folder in pkgs_dirs: if ( os.path.exists(os.path.join(folder, dist)) diff --git a/conda_build/utils.py b/conda_build/utils.py index cc7dcdf44f..293979d8c3 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -70,7 +70,6 @@ from contextlib import ExitStack # noqa: F401 from glob import glob -from conda.api import PackageCacheData # noqa from conda.base.constants import KNOWN_SUBDIRS from conda.core.prefix_data import PrefixData from conda.models.records import PrefixRecord From 44680fb038ef75d56a35bb48fa619a5e53e20888 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 00:04:29 +0100 Subject: [PATCH 11/61] Some Dist removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 20 ---------------- conda_build/conda_interface.py | 23 ------------------- conda_build/inspect_pkg.py | 6 ----- tests/test_conda_interface.py | 9 -------- 4 files changed, 58 deletions(-) delete mode 100644 tests/test_conda_interface.py diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 77be9b90e9..1e5d87fc99 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -38,23 +38,3 @@ def get_index( channel_urls, prepend, platform, use_local, use_cache, unknown, prefix ) return {Dist(prec): prec for prec in index.values()} - - -def linked_data(prefix, ignore_channels=False): - """Return a dictionary of the linked packages in prefix.""" - pd = _PrefixData(prefix) - return { - Dist(prefix_record): prefix_record - for prefix_record in pd._prefix_records.values() - } - - -def linked(prefix, ignore_channels=False): - """Return the Dists of linked packages in prefix.""" - conda_package_types = _PackageType.conda_package_types() - ld = linked_data(prefix, ignore_channels=ignore_channels).items() - return { - dist - for dist, prefix_rec in ld - if prefix_rec.package_type in conda_package_types - } diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 04f492f8c5..2c61582cc9 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -151,27 +151,6 @@ def which_prefix(path: str | os.PathLike | Path) -> Path: raise RuntimeError("could not determine conda prefix from: %s" % path) -@deprecated("3.28.0", "4.0.0") -def get_installed_version(prefix, pkgs): - """ - Primarily used by conda-forge, but may be useful in general for checking when - a package needs to be updated - """ - from conda_build.utils import ensure_list - - pkgs = ensure_list(pkgs) - linked_pkgs = linked(prefix) - versions = {} - for pkg in pkgs: - vers_inst = [ - dist.split("::", 1)[-1].rsplit("-", 2)[1] - for dist in linked_pkgs - if dist.split("::", 1)[-1].rsplit("-", 2)[0] == pkg - ] - versions[pkg] = vers_inst[0] if len(vers_inst) == 1 else None - return versions - - # When deactivating envs (e.g. switching from root to build/test) this env var is used, # except the PR that removed this has been reverted (for now) and Windows doesn't need it. env_path_backup_var_exists = os.environ.get("CONDA_PATH_BACKUP", None) @@ -184,6 +163,4 @@ def get_installed_version(prefix, pkgs): execute_actions, get_index, install_actions, - linked, - linked_data, ) diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index eac899f464..bd337f8cd2 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -24,7 +24,6 @@ display_actions, get_index, install_actions, - linked_data, specs_from_args, ) from conda_build.os_utils.ldd import ( @@ -249,11 +248,6 @@ def test_installable(channel="defaults"): return success -@deprecated("3.28.0", "4.0.0") -def _installed(prefix: str | os.PathLike | Path) -> dict[str, Dist]: - return {dist.name: dist for dist in linked_data(str(prefix))} - - def _underlined_text(text): return str(text) + "\n" + "-" * len(str(text)) + "\n\n" diff --git a/tests/test_conda_interface.py b/tests/test_conda_interface.py deleted file mode 100644 index 1c13d6faf9..0000000000 --- a/tests/test_conda_interface.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (C) 2014 Anaconda, Inc -# SPDX-License-Identifier: BSD-3-Clause -from conda_build import conda_interface as ci - - -def test_get_installed_version(): - versions = ci.get_installed_version(ci.root_dir, "conda") - assert versions.get("conda") - assert ci.VersionOrder(versions.get("conda")) From 2e33ab53e50c9ce6bf833ac2d11de26bd0cf4a34 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 00:33:01 +0100 Subject: [PATCH 12/61] Some Dist removal Signed-off-by: Marcel Bargull --- conda_build/post.py | 49 -------------------------------------------- conda_build/utils.py | 21 ++----------------- 2 files changed, 2 insertions(+), 68 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index 93c761d3d0..f42d74007d 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -773,38 +773,6 @@ def library_nature( return "non-library" -@deprecated( - "3.28.0", - "4.0.0", - addendum="Query `conda.core.prefix_data.PrefixData` instead.", -) -def dists_from_names(names: Iterable[str], prefix: str | os.PathLike | Path): - from conda_build.utils import linked_data_no_multichannels - - names = utils.ensure_list(names) - return [prec for prec in linked_data_no_multichannels(prefix) if prec.name in names] - - -@deprecated( - "3.28.0", - "4.0.0", - addendum="Use `conda.models.records.PrefixRecord` instead.", -) -class FakeDist: - def __init__(self, name, version, build_number, build_str, channel, files): - self.name = name - self.quad = [name] - self.version = version - self.build_number = build_number - self.build_string = build_str - self.channel = channel - self.files = files - - def get(self, name): - if name == "files": - return self.files - - # This is really just a small, fixed sysroot and it is rooted at ''. `libcrypto.0.9.8.dylib` should not be in it IMHO. DEFAULT_MAC_WHITELIST = [ "/opt/X11/", @@ -1018,23 +986,6 @@ def _map_file_to_package( return prefix_owners, contains_dsos, contains_static_libs, all_lib_exports -@deprecated( - "3.28.0", "4.0.0", addendum="Use `conda.models.records.PrefixRecord` instead." -) -def _get_fake_pkg_dist(pkg_name, pkg_version, build_str, build_number, channel, files): - return ( - FakeDist( - pkg_name, - str(pkg_version), - build_number, - build_str, - channel, - files, - ), - f"{pkg_name}-{pkg_version}-{build_str}", - ) - - def _print_msg(errors, text, verbose): if text.startswith(" ERROR"): errors.append(text) diff --git a/conda_build/utils.py b/conda_build/utils.py index 293979d8c3..afd80b64b8 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -81,7 +81,6 @@ from .conda_interface import ( # noqa CondaHTTPError, - Dist, # noqa MatchSpec, StringIO, # noqa TemporaryDirectory, @@ -2008,13 +2007,11 @@ def match_peer_job(target_matchspec, other_m, this_m=None): for any keys that are shared between target_variant and m.config.variant""" name, version, build = other_m.name(), other_m.version(), "" matchspec_matches = target_matchspec.match( - Dist( + PackageRecord( name=name, - dist_name=f"{name}-{version}-{build}", version=version, - build_string=build, + build=build, build_number=other_m.build_number(), - channel=None, ) ) @@ -2164,20 +2161,6 @@ def download_channeldata(channel_url): return data -def linked_data_no_multichannels( - prefix: str | os.PathLike | Path, -) -> dict[Dist, PrefixRecord]: - """ - Return a dictionary of the linked packages in prefix, with correct channels, hopefully. - cc @kalefranz. - """ - prefix = Path(prefix) - return { - Dist.from_string(prec.fn, channel_override=prec.channel.name): prec - for prec in PrefixData(str(prefix)).iter_records() - } - - def shutil_move_more_retrying(src, dest, debug_name): log = get_logger(__name__) log.info(f"Renaming {debug_name} directory '{src}' to '{dest}'") From a7febea7bdb96aa7c28f12ebe8da2cff6859dc30 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 00:34:52 +0100 Subject: [PATCH 13/61] Some Dist removal Signed-off-by: Marcel Bargull --- conda_build/inspect_pkg.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index bd337f8cd2..6dfe6e98d9 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -20,7 +20,6 @@ from conda.resolve import MatchSpec from conda_build.conda_interface import ( - Dist, display_actions, get_index, install_actions, @@ -45,17 +44,6 @@ from .utils import on_mac, on_win -@deprecated("3.28.0", "4.0.0") -@lru_cache(maxsize=None) -def dist_files(prefix: str | os.PathLike | Path, dist: Dist) -> set[str]: - if (prec := PrefixData(prefix).get(dist.name, None)) is None: - return set() - elif MatchSpec(dist).match(prec): - return set(prec["files"]) - else: - return set() - - @deprecated.argument("3.28.0", "4.0.0", "avoid_canonical_channel_name") def which_package( path: str | os.PathLike | Path, From ff3be6a44dcf1a80cc3e449b81c52b7a6af36f9c Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 12:04:57 +0100 Subject: [PATCH 14/61] Prepare some Dist removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 24 +++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 6507049ae1..ebf09a1cb3 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -44,20 +44,24 @@ def __call__(cls, *args, **kwargs): elif isinstance(value, Dist): dist = value elif isinstance(value, PackageRecord): - dist = Dist.from_string( + dist_kwargs = Dist._as_dict_from_string( value.fn, channel_override=value.channel.canonical_name ) + dist = super().__call__(**dist_kwargs) elif hasattr(value, "dist") and isinstance(value.dist, Dist): dist = value.dist elif isinstance(value, PackageInfo): - dist = Dist.from_string( + dist_kwargs = Dist._as_dict_from_string( value.repodata_record.fn, channel_override=value.channel.canonical_name, ) + dist = super().__call__(**dist_kwargs) elif isinstance(value, Channel): - dist = Dist.from_url(value.url()) + dist_kwargs = Dist._as_dict_from_url(value.url()) + dist = super().__call__(**dist_kwargs) else: - dist = Dist.from_string(value) + dist_kwargs = Dist._as_dict_from_string(value) + dist = super().__call__(**dist_kwargs) Dist._cache_[value] = dist return dist else: @@ -173,14 +177,14 @@ def to_match_spec(self): return MatchSpec(f"{self.channel}::{base}" if self.channel else base) @classmethod - def from_string(cls, string, channel_override=NULL): + def _as_dict_from_string(cls, string, channel_override=NULL): string = str(string) if is_url(string) and channel_override == NULL: - return cls.from_url(string) + return cls._as_dict_from_url(string) if string.endswith("@"): - return cls( + return dict( channel="@", name=string, version="", @@ -205,7 +209,7 @@ def from_string(cls, string, channel_override=NULL): # enforce dist format dist_details = cls.parse_dist_name(original_dist) - return cls( + return dict( channel=channel, name=dist_details.name, version=dist_details.version, @@ -251,7 +255,7 @@ def parse_dist_name(string): ) @classmethod - def from_url(cls, url): + def _as_dict_from_url(cls, url): assert is_url(url), url if ( not any(url.endswith(ext) for ext in CONDA_PACKAGE_EXTENSIONS) @@ -271,7 +275,7 @@ def from_url(cls, url): base_url = url_no_tarball.rsplit("/", 1)[0] if platform else url_no_tarball channel = Channel(base_url).canonical_name if platform else UNKNOWN_CHANNEL - return cls( + return dict( channel=channel, name=dist_details.name, version=dist_details.version, From 8e85b513ab61e7e7da2442f6f85de59c75f21c7e Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 12:20:26 +0100 Subject: [PATCH 15/61] Some Dist removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 105 ++++++---------------- 1 file changed, 29 insertions(+), 76 deletions(-) diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index ebf09a1cb3..6b2c628416 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -36,36 +36,31 @@ class DistDetails(NamedTuple): class DistType(EntityType): - def __call__(cls, *args, **kwargs): - if len(args) == 1 and not kwargs: - value = args[0] - if value in Dist._cache_: - return Dist._cache_[value] - elif isinstance(value, Dist): - dist = value - elif isinstance(value, PackageRecord): - dist_kwargs = Dist._as_dict_from_string( - value.fn, channel_override=value.channel.canonical_name - ) - dist = super().__call__(**dist_kwargs) - elif hasattr(value, "dist") and isinstance(value.dist, Dist): - dist = value.dist - elif isinstance(value, PackageInfo): - dist_kwargs = Dist._as_dict_from_string( - value.repodata_record.fn, - channel_override=value.channel.canonical_name, - ) - dist = super().__call__(**dist_kwargs) - elif isinstance(value, Channel): - dist_kwargs = Dist._as_dict_from_url(value.url()) - dist = super().__call__(**dist_kwargs) - else: - dist_kwargs = Dist._as_dict_from_string(value) - dist = super().__call__(**dist_kwargs) - Dist._cache_[value] = dist - return dist + def __call__(cls, value): + if value in Dist._cache_: + return Dist._cache_[value] + elif isinstance(value, Dist): + dist = value + elif isinstance(value, PackageRecord): + dist_kwargs = Dist._as_dict_from_string( + value.fn, channel_override=value.channel.canonical_name + ) + dist = super().__call__(**dist_kwargs) + elif hasattr(value, "dist") and isinstance(value.dist, Dist): + raise NotImplementedError() + elif isinstance(value, PackageInfo): + dist_kwargs = Dist._as_dict_from_string( + value.repodata_record.fn, + channel_override=value.channel.canonical_name, + ) + dist = super().__call__(**dist_kwargs) + elif isinstance(value, Channel): + raise NotImplementedError() else: - return super().__call__(*args, **kwargs) + dist_kwargs = Dist._as_dict_from_string(value) + dist = super().__call__(**dist_kwargs) + Dist._cache_[value] = dist + return dist def strip_extension(original_dist): @@ -130,10 +125,6 @@ def to_package_ref(self): build_number=self.build_number, ) - @property - def full_name(self): - return self.__str__() - @property def build(self): return self.build_string @@ -142,10 +133,6 @@ def build(self): def subdir(self): return self.platform - @property - def pair(self): - return self.channel or DEFAULTS_CHANNEL_NAME, self.dist_name - @property def quad(self): # returns: name, version, build_string, channel @@ -155,27 +142,6 @@ def quad(self): def __str__(self): return f"{self.channel}::{self.dist_name}" if self.channel else self.dist_name - @property - def is_feature_package(self): - return self.dist_name.endswith("@") - - @property - def is_channel(self): - return bool(self.base_url and self.platform) - - def to_filename(self, extension=None): - if self.is_feature_package: - return self.dist_name - else: - return self.dist_name + self.fmt - - def to_matchspec(self): - return " ".join(self.quad[:3]) - - def to_match_spec(self): - base = "=".join(self.quad[:3]) - return MatchSpec(f"{self.channel}::{base}" if self.channel else base) - @classmethod def _as_dict_from_string(cls, string, channel_override=NULL): string = str(string) @@ -287,33 +253,23 @@ def _as_dict_from_url(cls, url): fmt=dist_details.fmt, ) - def to_url(self): - if not self.base_url: - return None - filename = self.dist_name + self.fmt - return ( - join_url(self.base_url, self.platform, filename) - if self.platform - else join_url(self.base_url, filename) - ) - def __key__(self): return self.channel, self.dist_name def __lt__(self, other): - assert isinstance(other, self.__class__) + raise NotImplementedError() return self.__key__() < other.__key__() def __gt__(self, other): - assert isinstance(other, self.__class__) + raise NotImplementedError() return self.__key__() > other.__key__() def __le__(self, other): - assert isinstance(other, self.__class__) + raise NotImplementedError() return self.__key__() <= other.__key__() def __ge__(self, other): - assert isinstance(other, self.__class__) + raise NotImplementedError() return self.__key__() >= other.__key__() def __hash__(self): @@ -339,13 +295,10 @@ def rsplit(self, sep=None, maxsplit=-1): name = f"{self.channel}::{self.quad[0]}" if self.channel else self.quad[0] return name, self.quad[1], self.quad[2] - def startswith(self, match): - return self.dist_name.startswith(match) - def __contains__(self, item): item = strip_extension(ensure_text_type(item)) return item in self.__str__() @property def fn(self): - return self.to_filename() + return self.to_package_ref().fn From 96f121e950d4c3f2b631622773f0d8320fe834e4 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 13:02:56 +0100 Subject: [PATCH 16/61] Some Dist removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 75 +++++++---------------- 1 file changed, 23 insertions(+), 52 deletions(-) diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 6b2c628416..bf0e4f5638 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -29,7 +29,7 @@ class DistDetails(NamedTuple): name: str version: str - build_string: str + build: str build_number: str dist_name: str fmt: str @@ -49,11 +49,7 @@ def __call__(cls, value): elif hasattr(value, "dist") and isinstance(value.dist, Dist): raise NotImplementedError() elif isinstance(value, PackageInfo): - dist_kwargs = Dist._as_dict_from_string( - value.repodata_record.fn, - channel_override=value.channel.canonical_name, - ) - dist = super().__call__(**dist_kwargs) + raise NotImplementedError() elif isinstance(value, Channel): raise NotImplementedError() else: @@ -85,11 +81,11 @@ class Dist(Entity, metaclass=DistType): name = StringField(immutable=True) fmt = StringField(immutable=True) version = StringField(immutable=True) - build_string = StringField(immutable=True) + build = StringField(immutable=True) build_number = IntegerField(immutable=True) base_url = StringField(required=False, nullable=True, immutable=True) - platform = StringField(required=False, nullable=True, immutable=True) + subdir = StringField(required=False, nullable=True, immutable=True) def __init__( self, @@ -97,10 +93,10 @@ def __init__( dist_name=None, name=None, version=None, - build_string=None, + build=None, build_number=None, base_url=None, - platform=None, + subdir=None, fmt=".tar.bz2", ): super().__init__( @@ -108,37 +104,23 @@ def __init__( dist_name=dist_name, name=name, version=version, - build_string=build_string, + build=build, build_number=build_number, base_url=base_url, - platform=platform, + subdir=subdir, fmt=fmt, ) def to_package_ref(self): return PackageRecord( channel=self.channel, - subdir=self.platform, + subdir=self.subdir, name=self.name, version=self.version, - build=self.build_string, + build=self.build, build_number=self.build_number, ) - @property - def build(self): - return self.build_string - - @property - def subdir(self): - return self.platform - - @property - def quad(self): - # returns: name, version, build_string, channel - parts = self.dist_name.rsplit("-", 2) + ["", ""] - return parts[0], parts[1], parts[2], self.channel or DEFAULTS_CHANNEL_NAME - def __str__(self): return f"{self.channel}::{self.dist_name}" if self.channel else self.dist_name @@ -150,14 +132,7 @@ def _as_dict_from_string(cls, string, channel_override=NULL): return cls._as_dict_from_url(string) if string.endswith("@"): - return dict( - channel="@", - name=string, - version="", - build_string="", - build_number=0, - dist_name=string, - ) + raise NotImplementedError() REGEX_STR = ( r"(?:([^\s\[\]]+)::)?" # optional channel @@ -179,7 +154,7 @@ def _as_dict_from_string(cls, string, channel_override=NULL): channel=channel, name=dist_details.name, version=dist_details.version, - build_string=dist_details.build_string, + build=dist_details.build, build_number=dist_details.build_number, dist_name=original_dist, fmt=fmt, @@ -202,17 +177,17 @@ def parse_dist_name(string): name = parts[0] version = parts[1] - build_string = parts[2] if len(parts) >= 3 else "" + build = parts[2] if len(parts) >= 3 else "" build_number_as_string = "".join( filter( lambda x: x.isdigit(), - (build_string.rsplit("_")[-1] if build_string else "0"), + (build.rsplit("_")[-1] if build else "0"), ) ) build_number = int(build_number_as_string) if build_number_as_string else 0 return DistDetails( - name, version, build_string, build_number, dist_name, fmt + name, version, build, build_number, dist_name, fmt ) except: @@ -232,24 +207,24 @@ def _as_dict_from_url(cls, url): dist_details = cls.parse_dist_name(url) if "::" in url: url_no_tarball = url.rsplit("::", 1)[0] - platform = context.subdir + subdir = context.subdir base_url = url_no_tarball.split("::")[0] channel = str(Channel(base_url)) else: url_no_tarball = url.rsplit("/", 1)[0] - platform = has_platform(url_no_tarball, context.known_subdirs) - base_url = url_no_tarball.rsplit("/", 1)[0] if platform else url_no_tarball - channel = Channel(base_url).canonical_name if platform else UNKNOWN_CHANNEL + subdir = has_platform(url_no_tarball, context.known_subdirs) + base_url = url_no_tarball.rsplit("/", 1)[0] if subdir else url_no_tarball + channel = Channel(base_url).canonical_name if subdir else UNKNOWN_CHANNEL return dict( channel=channel, name=dist_details.name, version=dist_details.version, - build_string=dist_details.build_string, + build=dist_details.build, build_number=dist_details.build_number, dist_name=dist_details.dist_name, base_url=base_url, - platform=platform, + subdir=subdir, fmt=dist_details.fmt, ) @@ -286,14 +261,10 @@ def __ne__(self, other): # ############ conda-build compatibility ################ def split(self, sep=None, maxsplit=-1): - assert sep == "::" - return [self.channel, self.dist_name] if self.channel else [self.dist_name] + raise NotImplementedError() def rsplit(self, sep=None, maxsplit=-1): - assert sep == "-" - assert maxsplit == 2 - name = f"{self.channel}::{self.quad[0]}" if self.channel else self.quad[0] - return name, self.quad[1], self.quad[2] + raise NotImplementedError() def __contains__(self, item): item = strip_extension(ensure_text_type(item)) From b4f5a8ca02e3608a9acf63b03897483e468f4f35 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 13:14:30 +0100 Subject: [PATCH 17/61] Prepare some Dist removal; hoist parsing code Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 238 +++++++++++----------- 1 file changed, 119 insertions(+), 119 deletions(-) diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index bf0e4f5638..625590373c 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -35,6 +35,122 @@ class DistDetails(NamedTuple): fmt: str +def _strip_extension(original_dist): + for ext in CONDA_PACKAGE_EXTENSIONS: + if original_dist.endswith(ext): + original_dist = original_dist[: -len(ext)] + return original_dist + + +def _split_extension(original_dist): + stripped = _strip_extension(original_dist) + return stripped, original_dist[len(stripped) :] + + +def _parse_dist_name(string): + original_string = string + try: + string = ensure_text_type(string) + no_fmt_string, fmt = _split_extension(string) + + # remove any directory or channel information + if "::" in no_fmt_string: + dist_name = no_fmt_string.rsplit("::", 1)[-1] + else: + dist_name = no_fmt_string.rsplit("/", 1)[-1] + + parts = dist_name.rsplit("-", 2) + + name = parts[0] + version = parts[1] + build = parts[2] if len(parts) >= 3 else "" + build_number_as_string = "".join( + filter( + lambda x: x.isdigit(), + (build.rsplit("_")[-1] if build else "0"), + ) + ) + build_number = int(build_number_as_string) if build_number_as_string else 0 + + return DistDetails( + name, version, build, build_number, dist_name, fmt + ) + + except: + raise CondaError( + "dist_name is not a valid conda package: %s" % original_string + ) + + +def _as_dict_from_string(string, channel_override=NULL): + string = str(string) + + if is_url(string) and channel_override == NULL: + return _as_dict_from_url(string) + + if string.endswith("@"): + raise NotImplementedError() + + REGEX_STR = ( + r"(?:([^\s\[\]]+)::)?" # optional channel + r"([^\s\[\]]+)" # 3.x dist + r"(?:\[([a-zA-Z0-9_-]+)\])?" # with_features_depends + ) + channel, original_dist, w_f_d = re.search(REGEX_STR, string).groups() + + original_dist, fmt = _split_extension(original_dist) + + if channel_override != NULL: + channel = channel_override + if not channel: + channel = UNKNOWN_CHANNEL + + # enforce dist format + dist_details = _parse_dist_name(original_dist) + return dict( + channel=channel, + name=dist_details.name, + version=dist_details.version, + build=dist_details.build, + build_number=dist_details.build_number, + dist_name=original_dist, + fmt=fmt, + ) + + +def _as_dict_from_url(url): + assert is_url(url), url + if ( + not any(url.endswith(ext) for ext in CONDA_PACKAGE_EXTENSIONS) + and "::" not in url + ): + raise CondaError("url '%s' is not a conda package" % url) + + dist_details = _parse_dist_name(url) + if "::" in url: + url_no_tarball = url.rsplit("::", 1)[0] + subdir = context.subdir + base_url = url_no_tarball.split("::")[0] + channel = str(Channel(base_url)) + else: + url_no_tarball = url.rsplit("/", 1)[0] + subdir = has_platform(url_no_tarball, context.known_subdirs) + base_url = url_no_tarball.rsplit("/", 1)[0] if subdir else url_no_tarball + channel = Channel(base_url).canonical_name if subdir else UNKNOWN_CHANNEL + + return dict( + channel=channel, + name=dist_details.name, + version=dist_details.version, + build=dist_details.build, + build_number=dist_details.build_number, + dist_name=dist_details.dist_name, + base_url=base_url, + subdir=subdir, + fmt=dist_details.fmt, + ) + + class DistType(EntityType): def __call__(cls, value): if value in Dist._cache_: @@ -42,7 +158,7 @@ def __call__(cls, value): elif isinstance(value, Dist): dist = value elif isinstance(value, PackageRecord): - dist_kwargs = Dist._as_dict_from_string( + dist_kwargs = _as_dict_from_string( value.fn, channel_override=value.channel.canonical_name ) dist = super().__call__(**dist_kwargs) @@ -53,24 +169,12 @@ def __call__(cls, value): elif isinstance(value, Channel): raise NotImplementedError() else: - dist_kwargs = Dist._as_dict_from_string(value) + dist_kwargs = _as_dict_from_string(value) dist = super().__call__(**dist_kwargs) Dist._cache_[value] = dist return dist -def strip_extension(original_dist): - for ext in CONDA_PACKAGE_EXTENSIONS: - if original_dist.endswith(ext): - original_dist = original_dist[: -len(ext)] - return original_dist - - -def split_extension(original_dist): - stripped = strip_extension(original_dist) - return stripped, original_dist[len(stripped) :] - - class Dist(Entity, metaclass=DistType): _cache_ = {} _lazy_validate = True @@ -124,110 +228,6 @@ def to_package_ref(self): def __str__(self): return f"{self.channel}::{self.dist_name}" if self.channel else self.dist_name - @classmethod - def _as_dict_from_string(cls, string, channel_override=NULL): - string = str(string) - - if is_url(string) and channel_override == NULL: - return cls._as_dict_from_url(string) - - if string.endswith("@"): - raise NotImplementedError() - - REGEX_STR = ( - r"(?:([^\s\[\]]+)::)?" # optional channel - r"([^\s\[\]]+)" # 3.x dist - r"(?:\[([a-zA-Z0-9_-]+)\])?" # with_features_depends - ) - channel, original_dist, w_f_d = re.search(REGEX_STR, string).groups() - - original_dist, fmt = split_extension(original_dist) - - if channel_override != NULL: - channel = channel_override - if not channel: - channel = UNKNOWN_CHANNEL - - # enforce dist format - dist_details = cls.parse_dist_name(original_dist) - return dict( - channel=channel, - name=dist_details.name, - version=dist_details.version, - build=dist_details.build, - build_number=dist_details.build_number, - dist_name=original_dist, - fmt=fmt, - ) - - @staticmethod - def parse_dist_name(string): - original_string = string - try: - string = ensure_text_type(string) - no_fmt_string, fmt = split_extension(string) - - # remove any directory or channel information - if "::" in no_fmt_string: - dist_name = no_fmt_string.rsplit("::", 1)[-1] - else: - dist_name = no_fmt_string.rsplit("/", 1)[-1] - - parts = dist_name.rsplit("-", 2) - - name = parts[0] - version = parts[1] - build = parts[2] if len(parts) >= 3 else "" - build_number_as_string = "".join( - filter( - lambda x: x.isdigit(), - (build.rsplit("_")[-1] if build else "0"), - ) - ) - build_number = int(build_number_as_string) if build_number_as_string else 0 - - return DistDetails( - name, version, build, build_number, dist_name, fmt - ) - - except: - raise CondaError( - "dist_name is not a valid conda package: %s" % original_string - ) - - @classmethod - def _as_dict_from_url(cls, url): - assert is_url(url), url - if ( - not any(url.endswith(ext) for ext in CONDA_PACKAGE_EXTENSIONS) - and "::" not in url - ): - raise CondaError("url '%s' is not a conda package" % url) - - dist_details = cls.parse_dist_name(url) - if "::" in url: - url_no_tarball = url.rsplit("::", 1)[0] - subdir = context.subdir - base_url = url_no_tarball.split("::")[0] - channel = str(Channel(base_url)) - else: - url_no_tarball = url.rsplit("/", 1)[0] - subdir = has_platform(url_no_tarball, context.known_subdirs) - base_url = url_no_tarball.rsplit("/", 1)[0] if subdir else url_no_tarball - channel = Channel(base_url).canonical_name if subdir else UNKNOWN_CHANNEL - - return dict( - channel=channel, - name=dist_details.name, - version=dist_details.version, - build=dist_details.build, - build_number=dist_details.build_number, - dist_name=dist_details.dist_name, - base_url=base_url, - subdir=subdir, - fmt=dist_details.fmt, - ) - def __key__(self): return self.channel, self.dist_name @@ -267,7 +267,7 @@ def rsplit(self, sep=None, maxsplit=-1): raise NotImplementedError() def __contains__(self, item): - item = strip_extension(ensure_text_type(item)) + item = _strip_extension(ensure_text_type(item)) return item in self.__str__() @property From 8fac6ca6348ebf0546ce6cb9be84ae14559f6363 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 13:19:35 +0100 Subject: [PATCH 18/61] Only allow new Dist from PackageRecord or str Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 28 +++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 625590373c..503de0abce 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -152,25 +152,23 @@ def _as_dict_from_url(url): class DistType(EntityType): - def __call__(cls, value): - if value in Dist._cache_: - return Dist._cache_[value] - elif isinstance(value, Dist): - dist = value - elif isinstance(value, PackageRecord): + def _make_dist(cls, value): + if isinstance(value, Dist): + return dist + if isinstance(value, PackageRecord): dist_kwargs = _as_dict_from_string( value.fn, channel_override=value.channel.canonical_name ) - dist = super().__call__(**dist_kwargs) - elif hasattr(value, "dist") and isinstance(value.dist, Dist): - raise NotImplementedError() - elif isinstance(value, PackageInfo): - raise NotImplementedError() - elif isinstance(value, Channel): - raise NotImplementedError() - else: + return super().__call__(**dist_kwargs) + if isinstance(value, str): dist_kwargs = _as_dict_from_string(value) - dist = super().__call__(**dist_kwargs) + return super().__call__(**dist_kwargs) + raise NotImplementedError() + + def __call__(cls, value): + if value in Dist._cache_: + return Dist._cache_[value] + dist = cls._make_dist(value) Dist._cache_[value] = dist return dist From 77873d46b9533783d1996bb9f380ca256b567861 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 14:05:24 +0100 Subject: [PATCH 19/61] Some Dist removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 58 ++++------------------- 1 file changed, 10 insertions(+), 48 deletions(-) diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 503de0abce..925dea3e60 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -6,7 +6,6 @@ from .conda_imports import ( CONDA_PACKAGE_EXTENSIONS, - DEFAULTS_CHANNEL_NAME, NULL, UNKNOWN_CHANNEL, CondaError, @@ -14,15 +13,9 @@ Entity, EntityType, IntegerField, - MatchSpec, - PackageInfo, PackageRecord, StringField, - context, - ensure_text_type, - has_platform, is_url, - join_url, ) @@ -50,7 +43,6 @@ def _split_extension(original_dist): def _parse_dist_name(string): original_string = string try: - string = ensure_text_type(string) no_fmt_string, fmt = _split_extension(string) # remove any directory or channel information @@ -83,10 +75,8 @@ def _parse_dist_name(string): def _as_dict_from_string(string, channel_override=NULL): - string = str(string) - if is_url(string) and channel_override == NULL: - return _as_dict_from_url(string) + raise NotImplementedError() if string.endswith("@"): raise NotImplementedError() @@ -118,39 +108,6 @@ def _as_dict_from_string(string, channel_override=NULL): ) -def _as_dict_from_url(url): - assert is_url(url), url - if ( - not any(url.endswith(ext) for ext in CONDA_PACKAGE_EXTENSIONS) - and "::" not in url - ): - raise CondaError("url '%s' is not a conda package" % url) - - dist_details = _parse_dist_name(url) - if "::" in url: - url_no_tarball = url.rsplit("::", 1)[0] - subdir = context.subdir - base_url = url_no_tarball.split("::")[0] - channel = str(Channel(base_url)) - else: - url_no_tarball = url.rsplit("/", 1)[0] - subdir = has_platform(url_no_tarball, context.known_subdirs) - base_url = url_no_tarball.rsplit("/", 1)[0] if subdir else url_no_tarball - channel = Channel(base_url).canonical_name if subdir else UNKNOWN_CHANNEL - - return dict( - channel=channel, - name=dist_details.name, - version=dist_details.version, - build=dist_details.build, - build_number=dist_details.build_number, - dist_name=dist_details.dist_name, - base_url=base_url, - subdir=subdir, - fmt=dist_details.fmt, - ) - - class DistType(EntityType): def _make_dist(cls, value): if isinstance(value, Dist): @@ -224,7 +181,7 @@ def to_package_ref(self): ) def __str__(self): - return f"{self.channel}::{self.dist_name}" if self.channel else self.dist_name + raise NotImplementedError() def __key__(self): return self.channel, self.dist_name @@ -265,9 +222,14 @@ def rsplit(self, sep=None, maxsplit=-1): raise NotImplementedError() def __contains__(self, item): - item = _strip_extension(ensure_text_type(item)) - return item in self.__str__() + def to_str(x): + return f"{x.channel}::{x.dist_name}" if x.channel else x.dist_name + + if isinstance(item, Dist): + item = to_str(item) + item = _strip_extension(item) + return item in to_str(self) @property def fn(self): - return self.to_package_ref().fn + raise NotImplementedError() From d153be7df53234fbe2457b455f1eb36508710328 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 15:28:55 +0100 Subject: [PATCH 20/61] Remove unused imports for Dist Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 6 +----- conda_build/_legacy_conda_imports/conda_imports.py | 8 ++------ conda_build/_legacy_conda_imports/dist.py | 10 ++++++---- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 1e5d87fc99..546a3b44f5 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -1,8 +1,4 @@ -from .conda_imports import ( - PackageType as _PackageType, - PrefixData as _PrefixData, - get_index as _get_index, -) +from .conda_imports import get_index as _get_index from .dist import Dist from .instructions import ( LINK, diff --git a/conda_build/_legacy_conda_imports/conda_imports.py b/conda_build/_legacy_conda_imports/conda_imports.py index 988f8089ea..85cbdea456 100644 --- a/conda_build/_legacy_conda_imports/conda_imports.py +++ b/conda_build/_legacy_conda_imports/conda_imports.py @@ -21,15 +21,11 @@ UNKNOWN_CHANNEL, ) from conda.base.context import stack_context_default -from conda.common.compat import ensure_text_type -from conda.common.constants import NULL from conda.common.io import dashlist, env_vars from conda.common.iterators import groupby_to_dict -from conda.common.url import has_platform, is_url, join_url +from conda.common.url import is_url from conda.core.index import LAST_CHANNEL_URLS, get_index from conda.core.link import PrefixSetup, UnlinkLinkTransaction from conda.core.prefix_data import PrefixData -from conda.exceptions import ArgumentError from conda.models.channel import prioritize_channels -from conda.models.enums import LinkType, PackageType -from conda.models.package_info import PackageInfo +from conda.models.enums import LinkType diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 925dea3e60..488167c501 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -6,7 +6,6 @@ from .conda_imports import ( CONDA_PACKAGE_EXTENSIONS, - NULL, UNKNOWN_CHANNEL, CondaError, Channel, @@ -74,8 +73,11 @@ def _parse_dist_name(string): ) -def _as_dict_from_string(string, channel_override=NULL): - if is_url(string) and channel_override == NULL: +_not_set = object() + + +def _as_dict_from_string(string, channel_override=_not_set): + if is_url(string) and channel_override == _not_set: raise NotImplementedError() if string.endswith("@"): @@ -90,7 +92,7 @@ def _as_dict_from_string(string, channel_override=NULL): original_dist, fmt = _split_extension(original_dist) - if channel_override != NULL: + if channel_override != _not_set: channel = channel_override if not channel: channel = UNKNOWN_CHANNEL From d18f541f689cb3e9abac71e54a3347b8423da8a1 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 12 Nov 2023 20:41:53 +0100 Subject: [PATCH 21/61] Some Dist removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 48 +++++++++++++---------- conda_build/conda_interface.py | 1 - conda_build/environ.py | 44 ++++++++++----------- 3 files changed, 50 insertions(+), 43 deletions(-) diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 488167c501..708f98084f 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -110,6 +110,33 @@ def _as_dict_from_string(string, channel_override=_not_set): ) +def package_ref_from_dist_string(dist_string): + dist_kwargs = _as_dict_from_string(dist_string) + return PackageRecord( + channel=dist_kwargs["channel"], + name=dist_kwargs["name"], + version=dist_kwargs["version"], + build=dist_kwargs["build"], + build_number=dist_kwargs["build_number"], + ) + + +def dist_string_contains(containing_dist_string, contained_dist_string): + contained_dist_string = _strip_extension(contained_dist_string) + return contained_dist_string in containing_dist_string + + +def dist_string_from_package_record(package_record, channel=None): + if channel is None: + channel = package_record.channel.canonical_name + dist_kwargs = _as_dict_from_string( + package_record.fn, channel_override=channel + ) + channel = dist_kwargs["channel"] + dist_name = dist_kwargs["dist_name"] + return f"{channel}::{dist_name}" if channel else dist_name + + class DistType(EntityType): def _make_dist(cls, value): if isinstance(value, Dist): @@ -119,9 +146,6 @@ def _make_dist(cls, value): value.fn, channel_override=value.channel.canonical_name ) return super().__call__(**dist_kwargs) - if isinstance(value, str): - dist_kwargs = _as_dict_from_string(value) - return super().__call__(**dist_kwargs) raise NotImplementedError() def __call__(cls, value): @@ -172,16 +196,6 @@ def __init__( fmt=fmt, ) - def to_package_ref(self): - return PackageRecord( - channel=self.channel, - subdir=self.subdir, - name=self.name, - version=self.version, - build=self.build, - build_number=self.build_number, - ) - def __str__(self): raise NotImplementedError() @@ -224,13 +238,7 @@ def rsplit(self, sep=None, maxsplit=-1): raise NotImplementedError() def __contains__(self, item): - def to_str(x): - return f"{x.channel}::{x.dist_name}" if x.channel else x.dist_name - - if isinstance(item, Dist): - item = to_str(item) - item = _strip_extension(item) - return item in to_str(self) + raise NotImplementedError() @property def fn(self): diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 2c61582cc9..31a623c20d 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -158,7 +158,6 @@ def which_prefix(path: str | os.PathLike | Path) -> Path: from ._legacy_conda_imports import ( LINK, PREFIX, - Dist, display_actions, execute_actions, get_index, diff --git a/conda_build/environ.py b/conda_build/environ.py index 90fd1b3b7f..d44580b5b4 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -1214,7 +1214,12 @@ def remove_existing_packages(dirs, fns, config): @deprecated("3.28.0", "4.0.0") -def clean_pkg_cache(dist, config): +def clean_pkg_cache(dist_string, config): + from ._legacy_conda_imports.dist import ( + dist_string_contains, + dist_string_from_package_record, + package_ref_from_dist_string, + ) locks = [] conda_log_level = logging.WARN @@ -1236,40 +1241,35 @@ def clean_pkg_cache(dist, config): # Conda does not seem to do a complete cleanup sometimes. This is supplemental. # Conda's cleanup is still necessary - it keeps track of its own in-memory # list of downloaded things. - class package_cache: - def __contains__(self, dist): - return bool( - PackageCacheData.first_writable().get(Dist(dist).to_package_ref(), None) - ) - - def keys(self): - return (Dist(v) for v in PackageCacheData.first_writable().values()) - - def __delitem__(self, dist): - PackageCacheData.first_writable().remove(Dist(dist).to_package_ref()) + package_cache_data = PackageCacheData.first_writable() + package_records = [ + package_ref_from_dist_string(pkg_id) + for pkg_id in [dist_string, "local::" + dist_string] + ] for folder in pkgs_dirs: if ( - os.path.exists(os.path.join(folder, dist)) - or os.path.exists(os.path.join(folder, dist + ".tar.bz2")) + os.path.exists(os.path.join(folder, dist_string)) + or os.path.exists(os.path.join(folder, dist_string + ".tar.bz2")) or any( - pkg_id in package_cache() for pkg_id in [dist, "local::" + dist] + bool(PackageCacheData.first_writable().get(package_record, None)) + for package_record in package_records ) ): log = utils.get_logger(__name__) log.debug( "Conda caching error: %s package remains in cache after removal", - dist, + dist_string, ) log.debug("manually removing to compensate") - cache = package_cache() - keys = [key for key in cache.keys() if dist in key] - for pkg_id in keys: - if pkg_id in cache: - del cache[pkg_id] + for cache_pkg_id in package_cache_data.values(): + cache_pkg_id_dist_string = dist_string_from_package_record(cache_pkg_id) + if dist_string_contains(cache_pkg_id_dist_string, dist_string): + if bool(package_cache_data.get(cache_pkg_id, None)): + package_cache_data.remove(cache_pkg_id) # Note that this call acquires the relevant locks, so this must be called # outside the lock context above. - remove_existing_packages(pkgs_dirs, [dist], config) + remove_existing_packages(pkgs_dirs, [dist_string], config) def get_pinned_deps(m, section): From 101ce306974148674fd0a26aebdaeab78fab8df0 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Tue, 14 Nov 2023 22:03:58 +0100 Subject: [PATCH 22/61] Prepare some Dist removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 546a3b44f5..de95e3fbbe 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -21,6 +21,23 @@ def display_actions( ) +class _TemporaryIndexWrapper: + def __init__(self, index): + self._internal_index = index + self._internal_dict = {Dist(prec): prec for prec in index.values()} + def __contains__(self, key): + raise NotImplementedError() + def __iter__(self, key): + raise NotImplementedError() + def get(self, key, fallback=None): + raise NotImplementedError() + def __getitem__(self, key): + ret = self._internal_dict.__getitem__(key) + return ret + def values(self): + return self._internal_index.values() + + def get_index( channel_urls=(), prepend=True, @@ -33,4 +50,4 @@ def get_index( index = _get_index( channel_urls, prepend, platform, use_local, use_cache, unknown, prefix ) - return {Dist(prec): prec for prec in index.values()} + return _TemporaryIndexWrapper(index) From df219ffd82ace52138f6ede94e3b7522ea6ff54c Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Wed, 15 Nov 2023 11:54:31 +0100 Subject: [PATCH 23/61] Remove Dist Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 11 +- conda_build/_legacy_conda_imports/dist.py | 113 ------------------ conda_build/_legacy_conda_imports/plan.py | 8 +- conda_build/build.py | 2 +- conda_build/environ.py | 3 +- conda_build/index.py | 2 +- conda_build/render.py | 22 +--- 7 files changed, 18 insertions(+), 143 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index de95e3fbbe..26682aa635 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -1,5 +1,4 @@ from .conda_imports import get_index as _get_index -from .dist import Dist from .instructions import ( LINK, PREFIX, @@ -24,18 +23,18 @@ def display_actions( class _TemporaryIndexWrapper: def __init__(self, index): self._internal_index = index - self._internal_dict = {Dist(prec): prec for prec in index.values()} + self._internal_dict = {prec: prec for prec in index.values()} def __contains__(self, key): raise NotImplementedError() - def __iter__(self, key): - raise NotImplementedError() + def __iter__(self): + return self._internal_dict.__iter__() def get(self, key, fallback=None): raise NotImplementedError() def __getitem__(self, key): ret = self._internal_dict.__getitem__(key) return ret - def values(self): - return self._internal_index.values() + def get_internal_index(self): + return self._internal_index def get_index( diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 708f98084f..477433ef7c 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -8,12 +8,7 @@ CONDA_PACKAGE_EXTENSIONS, UNKNOWN_CHANNEL, CondaError, - Channel, - Entity, - EntityType, - IntegerField, PackageRecord, - StringField, is_url, ) @@ -135,111 +130,3 @@ def dist_string_from_package_record(package_record, channel=None): channel = dist_kwargs["channel"] dist_name = dist_kwargs["dist_name"] return f"{channel}::{dist_name}" if channel else dist_name - - -class DistType(EntityType): - def _make_dist(cls, value): - if isinstance(value, Dist): - return dist - if isinstance(value, PackageRecord): - dist_kwargs = _as_dict_from_string( - value.fn, channel_override=value.channel.canonical_name - ) - return super().__call__(**dist_kwargs) - raise NotImplementedError() - - def __call__(cls, value): - if value in Dist._cache_: - return Dist._cache_[value] - dist = cls._make_dist(value) - Dist._cache_[value] = dist - return dist - - -class Dist(Entity, metaclass=DistType): - _cache_ = {} - _lazy_validate = True - - channel = StringField(required=False, nullable=True, immutable=True) - - dist_name = StringField(immutable=True) - name = StringField(immutable=True) - fmt = StringField(immutable=True) - version = StringField(immutable=True) - build = StringField(immutable=True) - build_number = IntegerField(immutable=True) - - base_url = StringField(required=False, nullable=True, immutable=True) - subdir = StringField(required=False, nullable=True, immutable=True) - - def __init__( - self, - channel, - dist_name=None, - name=None, - version=None, - build=None, - build_number=None, - base_url=None, - subdir=None, - fmt=".tar.bz2", - ): - super().__init__( - channel=channel, - dist_name=dist_name, - name=name, - version=version, - build=build, - build_number=build_number, - base_url=base_url, - subdir=subdir, - fmt=fmt, - ) - - def __str__(self): - raise NotImplementedError() - - def __key__(self): - return self.channel, self.dist_name - - def __lt__(self, other): - raise NotImplementedError() - return self.__key__() < other.__key__() - - def __gt__(self, other): - raise NotImplementedError() - return self.__key__() > other.__key__() - - def __le__(self, other): - raise NotImplementedError() - return self.__key__() <= other.__key__() - - def __ge__(self, other): - raise NotImplementedError() - return self.__key__() >= other.__key__() - - def __hash__(self): - # dists compare equal regardless of fmt, but fmt is taken into account for - # object identity - return hash((self.__key__(), self.fmt)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__key__() == other.__key__() - - def __ne__(self, other): - return not self.__eq__(other) - - # ############ conda-build compatibility ################ - - def split(self, sep=None, maxsplit=-1): - raise NotImplementedError() - - def rsplit(self, sep=None, maxsplit=-1): - raise NotImplementedError() - - def __contains__(self, item): - raise NotImplementedError() - - @property - def fn(self): - raise NotImplementedError() diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index df39c60cec..bf3b81a1b4 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -37,7 +37,6 @@ stack_context_default, ) -from .dist import Dist from .instructions import ( ACTION_CODES, LINK, @@ -289,8 +288,7 @@ def _inject_UNLINKLINKTRANSACTION(plan, index, prefix): # pragma: no cover (q for q, p in enumerate(plan) if p[0] in (LINK,)), -1 ) if first_unlink_link_idx >= 0: - grouped_instructions = groupby(lambda x: x[0], plan) - link_dists = tuple(Dist(d[1]) for d in grouped_instructions.get(LINK, ())) + link_dists = tuple(dist for action, dist in plan if action == LINK) link_dists = _handle_menuinst(link_dists) link_precs = tuple(index[d] for d in link_dists) @@ -377,11 +375,11 @@ def install_actions( solver_backend = context.plugin_manager.get_cached_solver_backend() solver = solver_backend(prefix, channels, subdirs, specs_to_add=specs) if index: - solver._index = {prec: prec for prec in index.values()} + solver._index = index.get_internal_index() txn = solver.solve_for_transaction(prune=prune, ignore_pinned=not pinned) prefix_setup = txn.prefix_setups[prefix] actions = get_blank_actions(prefix) - actions[LINK].extend(Dist(prec) for prec in prefix_setup.link_precs) + actions[LINK].extend(prec for prec in prefix_setup.link_precs) return actions diff --git a/conda_build/build.py b/conda_build/build.py index 134730138a..bb1e9db59e 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -3868,7 +3868,7 @@ def build_tree( meta, actions, "host", - package_subset=dep, + package_subset=[dep], require_files=True, ) # test that package, using the local channel so that our new diff --git a/conda_build/environ.py b/conda_build/environ.py index d44580b5b4..7ac1f2a062 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -42,6 +42,7 @@ reset_context, root_dir, ) +from ._legacy_conda_imports.dist import dist_string_from_package_record from .deprecations import deprecated from .metadata import MetaData @@ -1290,6 +1291,6 @@ def get_pinned_deps(m, section): channel_urls=tuple(m.config.channel_urls), ) runtime_deps = [ - " ".join(link.dist_name.rsplit("-", 2)) for link in actions.get(LINK, []) + " ".join(dist_string_from_package_record(link).rsplit("-", 2)) for link in actions.get(LINK, []) ] return runtime_deps diff --git a/conda_build/index.py b/conda_build/index.py index 690673f0c9..86559bc051 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -230,7 +230,7 @@ def get_build_index( platform=subdir, ) - expanded_channels = {rec.channel for rec in cached_index.values()} + expanded_channels = {rec.channel for rec in cached_index.get_internal_index().values()} superchannel = {} # we need channeldata.json too, as it is a more reliable source of run_exports data diff --git a/conda_build/render.py b/conda_build/render.py index 693c1632c5..d4feaad4fb 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -46,6 +46,7 @@ pkgs_dirs, specs_from_url, ) +from ._legacy_conda_imports.dist import dist_string_from_package_record from .utils import CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2 # from conda_build.jinja_context import pin_subpackage_against_outputs @@ -93,7 +94,7 @@ def bldpkg_path(m): def actions_to_pins(actions): if LINK in actions: return [ - " ".join(spec.dist_name.split()[0].rsplit("-", 2)) + " ".join(dist_string_from_package_record(spec).split()[0].rsplit("-", 2)) for spec in actions[LINK] ] return [] @@ -359,7 +360,6 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files pkg_files = {} packages = actions.get(LINK, []) - package_subset = utils.ensure_list(package_subset) selected_packages = set() if package_subset: for pkg in package_subset: @@ -375,11 +375,7 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files packages = selected_packages for pkg in packages: - if hasattr(pkg, "dist_name"): - pkg_dist = pkg.dist_name - else: - pkg = strip_channel(pkg) - pkg_dist = pkg.split(" ")[0] + pkg_dist = strip_channel(dist_string_from_package_record(pkg)) pkg_loc = find_pkg_dir_or_file_in_pkgs_dirs( pkg_dist, m, files_only=require_files ) @@ -388,14 +384,8 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # TODO: this is a vile hack reaching into conda's internals. Replace with # proper conda API when available. if not pkg_loc: - try: - pkg_record = [_ for _ in index if _.dist_name == pkg_dist][0] - # the conda 4.4 API uses a single `link_prefs` kwarg - # whereas conda 4.3 used `index` and `link_dists` kwargs - pfe = ProgressiveFetchExtract(link_prefs=(index[pkg_record],)) - except TypeError: - # TypeError: __init__() got an unexpected keyword argument 'link_prefs' - pfe = ProgressiveFetchExtract(link_dists=[pkg], index=index) + pkg_record = [_ for _ in index if strip_channel(dist_string_from_package_record(_)) == pkg_dist][0] + pfe = ProgressiveFetchExtract(link_prefs=(index[pkg_record],)) with utils.LoggingContext(): pfe.execute() for pkg_dir in pkgs_dirs: @@ -432,7 +422,7 @@ def get_upstream_pins(m: MetaData, actions, env): run_exports = pkg_data.get("run_exports", {}).get(pkg.version, {}) if run_exports is None: loc, dist = execute_download_actions( - m, actions, env=env, package_subset=pkg + m, actions, env=env, package_subset=[pkg], )[pkg] run_exports = _read_specs_from_package(loc, dist) specs = _filter_run_exports(run_exports, ignore_list) From d306f96881acab888ee652b86fdb2e25356ea0f7 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Wed, 15 Nov 2023 12:06:50 +0100 Subject: [PATCH 24/61] Some post-Dist-removal cleanup Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 32 ++----------------- conda_build/_legacy_conda_imports/plan.py | 24 +++++++------- conda_build/environ.py | 2 +- conda_build/index.py | 2 +- conda_build/inspect_pkg.py | 4 +-- conda_build/render.py | 2 +- 6 files changed, 18 insertions(+), 48 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 26682aa635..9873fe3d79 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -4,39 +4,12 @@ PREFIX, ) from .plan import ( - display_actions as _display_actions, + display_actions execute_actions, install_actions, ) -def display_actions( - actions, index, show_channel_urls=None, specs_to_remove=(), specs_to_add=() -): - if LINK in actions: - actions[LINK] = [index[d] for d in actions[LINK]] - return _display_actions( - actions, show_channel_urls, specs_to_remove, specs_to_add - ) - - -class _TemporaryIndexWrapper: - def __init__(self, index): - self._internal_index = index - self._internal_dict = {prec: prec for prec in index.values()} - def __contains__(self, key): - raise NotImplementedError() - def __iter__(self): - return self._internal_dict.__iter__() - def get(self, key, fallback=None): - raise NotImplementedError() - def __getitem__(self, key): - ret = self._internal_dict.__getitem__(key) - return ret - def get_internal_index(self): - return self._internal_index - - def get_index( channel_urls=(), prepend=True, @@ -46,7 +19,6 @@ def get_index( unknown=None, prefix=None, ): - index = _get_index( + return _get_index( channel_urls, prepend, platform, use_local, use_cache, unknown, prefix ) - return _TemporaryIndexWrapper(index) diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index bf3b81a1b4..2efdc18f5d 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -288,10 +288,8 @@ def _inject_UNLINKLINKTRANSACTION(plan, index, prefix): # pragma: no cover (q for q, p in enumerate(plan) if p[0] in (LINK,)), -1 ) if first_unlink_link_idx >= 0: - link_dists = tuple(dist for action, dist in plan if action == LINK) - link_dists = _handle_menuinst(link_dists) - - link_precs = tuple(index[d] for d in link_dists) + link_precs = tuple(prec for action, prec in plan if action == LINK) + link_precs = _handle_menuinst(link_precs) pfe = ProgressiveFetchExtract(link_precs) pfe.prepare() @@ -305,25 +303,25 @@ def _inject_UNLINKLINKTRANSACTION(plan, index, prefix): # pragma: no cover return plan -def _handle_menuinst(link_dists): # pragma: no cover +def _handle_menuinst(link_precs): # pragma: no cover if not on_win: - return link_dists + return link_precs # Always link menuinst first/last on windows in case a subsequent # package tries to import it to create/remove a shortcut # link menuinst_idx = next( - (q for q, d in enumerate(link_dists) if d.name == "menuinst"), None + (q for q, d in enumerate(link_precs) if d.name == "menuinst"), None ) if menuinst_idx is not None: - link_dists = ( - *link_dists[menuinst_idx : menuinst_idx + 1], - *link_dists[:menuinst_idx], - *link_dists[menuinst_idx + 1 :], + link_precs = ( + *link_precs[menuinst_idx : menuinst_idx + 1], + *link_precs[:menuinst_idx], + *link_precs[menuinst_idx + 1 :], ) - return link_dists + return link_precs def install_actions( @@ -375,7 +373,7 @@ def install_actions( solver_backend = context.plugin_manager.get_cached_solver_backend() solver = solver_backend(prefix, channels, subdirs, specs_to_add=specs) if index: - solver._index = index.get_internal_index() + solver._index = index txn = solver.solve_for_transaction(prune=prune, ignore_pinned=not pinned) prefix_setup = txn.prefix_setups[prefix] actions = get_blank_actions(prefix) diff --git a/conda_build/environ.py b/conda_build/environ.py index 7ac1f2a062..ca0af56de2 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -1049,7 +1049,7 @@ def create_env( timeout=config.timeout, ) utils.trim_empty_keys(actions) - display_actions(actions, index) + display_actions(actions) if utils.on_win: for k, v in os.environ.items(): os.environ[k] = str(v) diff --git a/conda_build/index.py b/conda_build/index.py index 86559bc051..e3f12c8b60 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -230,7 +230,7 @@ def get_build_index( platform=subdir, ) - expanded_channels = {rec.channel for rec in cached_index.get_internal_index().values()} + expanded_channels = {rec.channel for rec in cached_index} superchannel = {} # we need channeldata.json too, as it is a more reliable source of run_exports data diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index 6dfe6e98d9..0134fb7971 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -109,7 +109,7 @@ def check_install( actions = install_actions( prefix, index, specs, pinned=False, minimal_hint=minimal_hint ) - display_actions(actions, index) + display_actions(actions) return actions finally: rm_rf(prefix) @@ -185,7 +185,7 @@ def test_installable(channel="defaults"): log.info("######## Testing platform %s ########", platform) channels = [channel] index = get_index(channel_urls=channels, prepend=False, platform=platform) - for _, rec in index.items(): + for rec in index: # If we give channels at the command line, only look at # packages from those channels (not defaults). if channel != "defaults" and rec.get("schannel", "defaults") == "defaults": diff --git a/conda_build/render.py b/conda_build/render.py index d4feaad4fb..7b08090252 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -385,7 +385,7 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # proper conda API when available. if not pkg_loc: pkg_record = [_ for _ in index if strip_channel(dist_string_from_package_record(_)) == pkg_dist][0] - pfe = ProgressiveFetchExtract(link_prefs=(index[pkg_record],)) + pfe = ProgressiveFetchExtract(link_prefs=(pkg_record,)) with utils.LoggingContext(): pfe.execute() for pkg_dir in pkgs_dirs: From 32d7d528e7bf25e7b087afe9adbc47625077ce9e Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Wed, 15 Nov 2023 12:15:47 +0100 Subject: [PATCH 25/61] Some post-Dist-removal cleanup Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 18 ++---------------- .../_legacy_conda_imports/instructions.py | 10 +++++----- conda_build/_legacy_conda_imports/plan.py | 19 ++++++++----------- conda_build/environ.py | 2 +- conda_build/render.py | 1 - 5 files changed, 16 insertions(+), 34 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 9873fe3d79..5bf0e4f7f4 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -1,24 +1,10 @@ -from .conda_imports import get_index as _get_index +from .conda_imports import get_index from .instructions import ( LINK, PREFIX, ) from .plan import ( - display_actions + display_actions, execute_actions, install_actions, ) - - -def get_index( - channel_urls=(), - prepend=True, - platform=None, - use_local=False, - use_cache=False, - unknown=None, - prefix=None, -): - return _get_index( - channel_urls, prepend, platform, use_local, use_cache, unknown, prefix - ) diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py index 691a289b32..9b0886db51 100644 --- a/conda_build/_legacy_conda_imports/instructions.py +++ b/conda_build/_legacy_conda_imports/instructions.py @@ -21,22 +21,22 @@ ) -def PREFIX_CMD(state, prefix): - state["prefix"] = prefix +def PREFIX_CMD(prefix): + pass -def PRINT_CMD(state, arg): # pragma: no cover +def PRINT_CMD(arg): # pragma: no cover if arg.startswith(("Linking packages",)): return getLogger("conda.stdout.verbose").info(arg) -def PROGRESSIVEFETCHEXTRACT_CMD(state, progressive_fetch_extract): # pragma: no cover +def PROGRESSIVEFETCHEXTRACT_CMD(progressive_fetch_extract): # pragma: no cover assert isinstance(progressive_fetch_extract, ProgressiveFetchExtract) progressive_fetch_extract.execute() -def UNLINKLINKTRANSACTION_CMD(state, arg): # pragma: no cover +def UNLINKLINKTRANSACTION_CMD(arg): # pragma: no cover unlink_link_transaction = arg assert isinstance(unlink_link_transaction, UnlinkLinkTransaction) unlink_link_transaction.execute() diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 2efdc18f5d..5b90e4f6b1 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -238,12 +238,12 @@ def format(s, pkg): # ---------------------------- Backwards compat for conda-build -------------------------- -def execute_actions(actions, index, verbose=False): # pragma: no cover - plan = _plan_from_actions(actions, index) - execute_instructions(plan, index, verbose) +def execute_actions(actions, verbose=False): # pragma: no cover + plan = _plan_from_actions(actions) + execute_instructions(plan, verbose) -def _plan_from_actions(actions, index): # pragma: no cover +def _plan_from_actions(actions): # pragma: no cover if OP_ORDER in actions and actions[OP_ORDER]: op_order = actions[OP_ORDER] @@ -277,12 +277,12 @@ def _plan_from_actions(actions, index): # pragma: no cover log.debug(f"appending value {arg} for action {op}") plan.append((op, arg)) - plan = _inject_UNLINKLINKTRANSACTION(plan, index, prefix) + plan = _inject_UNLINKLINKTRANSACTION(plan, prefix) return plan -def _inject_UNLINKLINKTRANSACTION(plan, index, prefix): # pragma: no cover +def _inject_UNLINKLINKTRANSACTION(plan, prefix): # pragma: no cover # this is only used for conda-build at this point first_unlink_link_idx = next( (q for q, p in enumerate(plan) if p[0] in (LINK,)), -1 @@ -390,21 +390,18 @@ def get_blank_actions(prefix): # pragma: no cover return actions -def execute_instructions(plan, index=None, verbose=False): +def execute_instructions(plan, verbose=False): """Execute the instructions in the plan :param plan: A list of (instruction, arg) tuples - :param index: The meta-data index :param verbose: verbose output """ log.debug("executing plan %s", plan) - state = {"i": None, "prefix": context.root_prefix, "index": index} - for instruction, arg in plan: log.debug(" %s(%r)", instruction, arg) cmd = commands[instruction] if callable(cmd): - cmd(state, arg) + cmd(arg) diff --git a/conda_build/environ.py b/conda_build/environ.py index ca0af56de2..a8a70b2a69 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -1055,7 +1055,7 @@ def create_env( os.environ[k] = str(v) with env_var("CONDA_QUIET", not config.verbose, reset_context): with env_var("CONDA_JSON", not config.verbose, reset_context): - execute_actions(actions, index) + execute_actions(actions) except ( SystemExit, PaddingError, diff --git a/conda_build/render.py b/conda_build/render.py index 7b08090252..9f6a6a686f 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -42,7 +42,6 @@ ProgressiveFetchExtract, TemporaryDirectory, UnsatisfiableError, - execute_actions, pkgs_dirs, specs_from_url, ) From fe066e9beecf01a3e1af9f2f148bca5ed47c98cd Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Wed, 15 Nov 2023 14:08:21 +0100 Subject: [PATCH 26/61] Dead code removal Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/plan.py | 160 +++------------------- 1 file changed, 22 insertions(+), 138 deletions(-) diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 5b90e4f6b1..76c5e77c79 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -53,30 +53,15 @@ # TODO: Remove conda/plan.py. This module should be almost completely deprecated now. -def display_actions( - actions, show_channel_urls=None, specs_to_remove=(), specs_to_add=() -): +def display_actions(actions): prefix = actions.get(PREFIX) builder = ["", "## Package Plan ##\n"] if prefix: builder.append(" environment location: %s" % prefix) builder.append("") - if specs_to_remove: - builder.append( - " removed specs: %s" - % dashlist(sorted(str(s) for s in specs_to_remove), indent=4) - ) - builder.append("") - if specs_to_add: - builder.append( - " added / updated specs: %s" - % dashlist(sorted(str(s) for s in specs_to_add), indent=4) - ) - builder.append("") print("\n".join(builder)) - if show_channel_urls is None: - show_channel_urls = context.show_channel_urls + show_channel_urls = context.show_channel_urls def channel_str(rec): if rec.get("schannel"): @@ -94,144 +79,43 @@ def channel_filt(s): return "" return s - # package -> [oldver-oldbuild, newver-newbuild] - packages = defaultdict(lambda: list(("", ""))) - features = defaultdict(lambda: list(("", ""))) - channels = defaultdict(lambda: list(("", ""))) - records = defaultdict(lambda: list((None, None))) - linktypes = {} + packages = defaultdict(lambda: "") + features = defaultdict(lambda: "") + channels = defaultdict(lambda: "") for prec in actions.get(LINK, []): assert isinstance(prec, PackageRecord) pkg = prec["name"] - channels[pkg][1] = channel_str(prec) - packages[pkg][1] = prec["version"] + "-" + prec["build"] - records[pkg][1] = prec - # TODO: this is a lie; may have to give this report after - # UnlinkLinkTransaction.verify() - linktypes[pkg] = LinkType.hardlink - features[pkg][1] = ",".join(prec.get("features") or ()) - - new = {p for p in packages if not packages[p][0]} - removed = {p for p in packages if not packages[p][1]} - # New packages are actually listed in the left-hand column, - # so let's move them over there - for pkg in new: - for var in (packages, features, channels, records): - var[pkg] = var[pkg][::-1] - - updated = set() - downgraded = set() - channeled = set() - oldfmt = {} - newfmt = {} - empty = True + channels[pkg] = channel_filt(channel_str(prec)) + packages[pkg] = prec["version"] + "-" + prec["build"] + features[pkg] = ",".join(prec.get("features") or ()) + + fmt = {} if packages: - empty = False maxpkg = max(len(p) for p in packages) + 1 - maxoldver = max(len(p[0]) for p in packages.values()) - maxnewver = max(len(p[1]) for p in packages.values()) - maxoldfeatures = max(len(p[0]) for p in features.values()) - maxnewfeatures = max(len(p[1]) for p in features.values()) - maxoldchannels = max(len(channel_filt(p[0])) for p in channels.values()) - maxnewchannels = max(len(channel_filt(p[1])) for p in channels.values()) + maxver = max(len(p) for p in packages.values()) + maxfeatures = max(len(p) for p in features.values()) + maxchannels = max(len(p) for p in channels.values()) for pkg in packages: # That's right. I'm using old-style string formatting to generate a # string with new-style string formatting. - oldfmt[pkg] = f"{{pkg:<{maxpkg}}} {{vers[0]:<{maxoldver}}}" - if maxoldchannels: - oldfmt[pkg] += " {channels[0]:<%s}" % maxoldchannels - if features[pkg][0]: - oldfmt[pkg] += " [{features[0]:<%s}]" % maxoldfeatures - - lt = LinkType(linktypes.get(pkg, LinkType.hardlink)) - lt = "" if lt == LinkType.hardlink else (" (%s)" % lt) - if pkg in removed or pkg in new: - oldfmt[pkg] += lt - continue - - newfmt[pkg] = "{vers[1]:<%s}" % maxnewver - if maxnewchannels: - newfmt[pkg] += " {channels[1]:<%s}" % maxnewchannels - if features[pkg][1]: - newfmt[pkg] += " [{features[1]:<%s}]" % maxnewfeatures - newfmt[pkg] += lt - - P0 = records[pkg][0] - P1 = records[pkg][1] - pri0 = P0.get("priority") - pri1 = P1.get("priority") - if pri0 is None or pri1 is None: - pri0 = pri1 = 1 - try: - if str(P1.version) == "custom": - newver = str(P0.version) != "custom" - oldver = not newver - else: - # <= here means that unchanged packages will be put in updated - N0 = normalized_version(P0.version) - N1 = normalized_version(P1.version) - newver = N0 < N1 - oldver = N0 > N1 - except TypeError: - newver = P0.version < P1.version - oldver = P0.version > P1.version - oldbld = P0.build_number > P1.build_number - newbld = P0.build_number < P1.build_number - if ( - context.channel_priority - and pri1 < pri0 - and (oldver or not newver and not newbld) - ): - channeled.add(pkg) - elif newver: - updated.add(pkg) - elif pri1 < pri0 and (oldver or not newver and oldbld): - channeled.add(pkg) - elif oldver: - downgraded.add(pkg) - elif not oldbld: - updated.add(pkg) - else: - downgraded.add(pkg) + fmt[pkg] = f"{{pkg:<{maxpkg}}} {{vers:<{maxver}}}" + if maxchannels: + fmt[pkg] += " {channel:<%s}" % maxchannels + if features[pkg]: + fmt[pkg] += " [{features:<%s}]" % maxfeatures - arrow = " --> " lead = " " * 4 def format(s, pkg): - chans = [channel_filt(c) for c in channels[pkg]] return lead + s.format( - pkg=pkg + ":", vers=packages[pkg], channels=chans, features=features[pkg] + pkg=pkg + ":", vers=packages[pkg], channel=channels[pkg], features=features[pkg] ) - if new: + if packages: print("\nThe following NEW packages will be INSTALLED:\n") - for pkg in sorted(new): - # New packages have been moved to the "old" column for display - print(format(oldfmt[pkg], pkg)) - - if removed: - print("\nThe following packages will be REMOVED:\n") - for pkg in sorted(removed): - print(format(oldfmt[pkg], pkg)) - - if updated: - print("\nThe following packages will be UPDATED:\n") - for pkg in sorted(updated): - print(format(oldfmt[pkg] + arrow + newfmt[pkg], pkg)) - - if channeled: - print( - "\nThe following packages will be SUPERSEDED by a higher-priority channel:\n" - ) - for pkg in sorted(channeled): - print(format(oldfmt[pkg] + arrow + newfmt[pkg], pkg)) - - if downgraded: - print("\nThe following packages will be DOWNGRADED:\n") - for pkg in sorted(downgraded): - print(format(oldfmt[pkg] + arrow + newfmt[pkg], pkg)) - + for pkg in sorted(packages): + print(format(fmt[pkg], pkg)) print() From 98fd4e9167c18bd7a19bf3f0e0c328bb6515736c Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Wed, 15 Nov 2023 14:15:26 +0100 Subject: [PATCH 27/61] Remove unused imports Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/conda_imports.py | 5 +---- conda_build/_legacy_conda_imports/plan.py | 4 ---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/conda_build/_legacy_conda_imports/conda_imports.py b/conda_build/_legacy_conda_imports/conda_imports.py index 85cbdea456..920b79b5e8 100644 --- a/conda_build/_legacy_conda_imports/conda_imports.py +++ b/conda_build/_legacy_conda_imports/conda_imports.py @@ -14,18 +14,15 @@ normalized_version, ) -from conda.auxlib.entity import Entity, EntityType, IntegerField, StringField from conda.base.constants import ( CONDA_PACKAGE_EXTENSIONS, DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL, ) from conda.base.context import stack_context_default -from conda.common.io import dashlist, env_vars -from conda.common.iterators import groupby_to_dict +from conda.common.io import env_vars from conda.common.url import is_url from conda.core.index import LAST_CHANNEL_URLS, get_index from conda.core.link import PrefixSetup, UnlinkLinkTransaction from conda.core.prefix_data import PrefixData from conda.models.channel import prioritize_channels -from conda.models.enums import LinkType diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 76c5e77c79..4007afacea 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -20,7 +20,6 @@ UNKNOWN_CHANNEL, Channel, IndexedSet, - LinkType, MatchSpec, PackageRecord, PrefixData, @@ -28,10 +27,7 @@ ProgressiveFetchExtract, UnlinkLinkTransaction, context, - dashlist, env_vars, - groupby_to_dict as groupby, - normalized_version, on_win, prioritize_channels, stack_context_default, From daf5ab824c75dc5b31731c882ba99640f0c09ad2 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Wed, 15 Nov 2023 20:54:47 +0100 Subject: [PATCH 28/61] Fix imports Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/conda_imports.py | 3 +-- conda_build/utils.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/_legacy_conda_imports/conda_imports.py b/conda_build/_legacy_conda_imports/conda_imports.py index 920b79b5e8..7df4d71d91 100644 --- a/conda_build/_legacy_conda_imports/conda_imports.py +++ b/conda_build/_legacy_conda_imports/conda_imports.py @@ -9,7 +9,6 @@ MatchSpec, PackageRecord, ProgressiveFetchExtract, - context, on_win, normalized_version, ) @@ -19,7 +18,7 @@ DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL, ) -from conda.base.context import stack_context_default +from conda.base.context import context, stack_context_default from conda.common.io import env_vars from conda.common.url import is_url from conda.core.index import LAST_CHANNEL_URLS, get_index diff --git a/conda_build/utils.py b/conda_build/utils.py index afd80b64b8..6e12bfe3ae 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -82,6 +82,7 @@ from .conda_interface import ( # noqa CondaHTTPError, MatchSpec, + PackageRecord, StringIO, # noqa TemporaryDirectory, VersionOrder, From 51bcfb64f27c92d9b57bcf386ed48bfa9e1adb52 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Sun, 19 Nov 2023 02:12:44 +0100 Subject: [PATCH 29/61] Fix dist_name never includes channel Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 4 +--- conda_build/render.py | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 477433ef7c..a6944cf20e 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -127,6 +127,4 @@ def dist_string_from_package_record(package_record, channel=None): dist_kwargs = _as_dict_from_string( package_record.fn, channel_override=channel ) - channel = dist_kwargs["channel"] - dist_name = dist_kwargs["dist_name"] - return f"{channel}::{dist_name}" if channel else dist_name + return dist_kwargs["dist_name"] diff --git a/conda_build/render.py b/conda_build/render.py index 9f6a6a686f..861d2b6c90 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -374,7 +374,7 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files packages = selected_packages for pkg in packages: - pkg_dist = strip_channel(dist_string_from_package_record(pkg)) + pkg_dist = dist_string_from_package_record(pkg) pkg_loc = find_pkg_dir_or_file_in_pkgs_dirs( pkg_dist, m, files_only=require_files ) @@ -383,12 +383,12 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # TODO: this is a vile hack reaching into conda's internals. Replace with # proper conda API when available. if not pkg_loc: - pkg_record = [_ for _ in index if strip_channel(dist_string_from_package_record(_)) == pkg_dist][0] + pkg_record = [_ for _ in index if dist_string_from_package_record(_) == pkg_dist][0] pfe = ProgressiveFetchExtract(link_prefs=(pkg_record,)) with utils.LoggingContext(): pfe.execute() for pkg_dir in pkgs_dirs: - _loc = join(pkg_dir, index.get(pkg, pkg).fn) + _loc = join(pkg_dir, pkg.fn) if isfile(_loc): pkg_loc = _loc break From 70e24a9e081d94bd3c2d8f4837822107db23334a Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 02:51:49 +0100 Subject: [PATCH 30/61] Avoid index manipulation in plan.install_actions Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/plan.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 4007afacea..cadfd083ea 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -253,7 +253,10 @@ def install_actions( solver_backend = context.plugin_manager.get_cached_solver_backend() solver = solver_backend(prefix, channels, subdirs, specs_to_add=specs) if index: - solver._index = index + # Solver can modify the index (e.g., Solver._prepare adds virtual + # package) => Copy index (just outer container, not deep copy) + # to conserve it. + solver._index = index.copy() txn = solver.solve_for_transaction(prune=prune, ignore_pinned=not pinned) prefix_setup = txn.prefix_setups[prefix] actions = get_blank_actions(prefix) From 9e17c5a4d865c8300f0c57b0590fe8ec79e345a8 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 03:21:53 +0100 Subject: [PATCH 31/61] Action plan operation order is always just [LINK] Signed-off-by: Marcel Bargull --- .../_legacy_conda_imports/instructions.py | 6 ---- conda_build/_legacy_conda_imports/plan.py | 34 ++++++------------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py index 9b0886db51..e345d85dba 100644 --- a/conda_build/_legacy_conda_imports/instructions.py +++ b/conda_build/_legacy_conda_imports/instructions.py @@ -14,12 +14,6 @@ UNLINKLINKTRANSACTION = "UNLINKLINKTRANSACTION" PROGRESSIVEFETCHEXTRACT = "PROGRESSIVEFETCHEXTRACT" -OP_ORDER = "op_order" - -ACTION_CODES = ( - LINK, -) - def PREFIX_CMD(prefix): pass diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index cadfd083ea..b5b272b83d 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -34,9 +34,7 @@ ) from .instructions import ( - ACTION_CODES, LINK, - OP_ORDER, PREFIX, PRINT, PROGRESSIVEFETCHEXTRACT, @@ -124,12 +122,6 @@ def execute_actions(actions, verbose=False): # pragma: no cover def _plan_from_actions(actions): # pragma: no cover - - if OP_ORDER in actions and actions[OP_ORDER]: - op_order = actions[OP_ORDER] - else: - op_order = ACTION_CODES - assert PREFIX in actions and actions[PREFIX] prefix = actions[PREFIX] plan = [(PREFIX, "%s" % prefix)] @@ -143,19 +135,16 @@ def _plan_from_actions(actions): # pragma: no cover # plan.append((UNLINKLINKTRANSACTION, unlink_link_transaction)) # return plan - log.debug(f"Adding plans for operations: {op_order}") - for op in op_order: - if op not in actions: - log.trace(f"action {op} not in actions") - continue - if not actions[op]: - log.trace(f"action {op} has None value") - continue - if "_" not in op: - plan.append((PRINT, "%sing packages ..." % op.capitalize())) - for arg in actions[op]: - log.debug(f"appending value {arg} for action {op}") - plan.append((op, arg)) + log.debug(f"Adding plans for operations: {[LINK]}") + if LINK not in actions: + log.trace(f"action {LINK} not in actions") + elif not actions[LINK]: + log.trace(f"action {LINK} has None value") + else: + plan.append((PRINT, "%sing packages ..." % LINK.capitalize())) + for arg in actions[LINK]: + log.debug(f"appending value {arg} for action {LINK}") + plan.append((LINK, arg)) plan = _inject_UNLINKLINKTRANSACTION(plan, prefix) @@ -267,9 +256,6 @@ def install_actions( def get_blank_actions(prefix): # pragma: no cover actions = defaultdict(list) actions[PREFIX] = prefix - actions[OP_ORDER] = ( - LINK, - ) return actions From cf6ea6300fc45178ebf0b4e2b5fdd44513d1ee4c Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 03:26:56 +0100 Subject: [PATCH 32/61] Link actions are never printed Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/instructions.py | 8 -------- conda_build/_legacy_conda_imports/plan.py | 2 -- 2 files changed, 10 deletions(-) diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py index e345d85dba..ddc5b2b387 100644 --- a/conda_build/_legacy_conda_imports/instructions.py +++ b/conda_build/_legacy_conda_imports/instructions.py @@ -9,7 +9,6 @@ # op codes PREFIX = "PREFIX" -PRINT = "PRINT" LINK = "LINK" UNLINKLINKTRANSACTION = "UNLINKLINKTRANSACTION" PROGRESSIVEFETCHEXTRACT = "PROGRESSIVEFETCHEXTRACT" @@ -19,12 +18,6 @@ def PREFIX_CMD(prefix): pass -def PRINT_CMD(arg): # pragma: no cover - if arg.startswith(("Linking packages",)): - return - getLogger("conda.stdout.verbose").info(arg) - - def PROGRESSIVEFETCHEXTRACT_CMD(progressive_fetch_extract): # pragma: no cover assert isinstance(progressive_fetch_extract, ProgressiveFetchExtract) progressive_fetch_extract.execute() @@ -39,7 +32,6 @@ def UNLINKLINKTRANSACTION_CMD(arg): # pragma: no cover # Map instruction to command (a python function) commands = { PREFIX: PREFIX_CMD, - PRINT: PRINT_CMD, LINK: None, UNLINKLINKTRANSACTION: UNLINKLINKTRANSACTION_CMD, PROGRESSIVEFETCHEXTRACT: PROGRESSIVEFETCHEXTRACT_CMD, diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index b5b272b83d..8c4b05a47e 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -36,7 +36,6 @@ from .instructions import ( LINK, PREFIX, - PRINT, PROGRESSIVEFETCHEXTRACT, UNLINKLINKTRANSACTION, commands, @@ -141,7 +140,6 @@ def _plan_from_actions(actions): # pragma: no cover elif not actions[LINK]: log.trace(f"action {LINK} has None value") else: - plan.append((PRINT, "%sing packages ..." % LINK.capitalize())) for arg in actions[LINK]: log.debug(f"appending value {arg} for action {LINK}") plan.append((LINK, arg)) From f6f6fa08dbde3f0a43928a61196a38c2a371901d Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 03:41:54 +0100 Subject: [PATCH 33/61] Inline one-off execute_/install_actions functions Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 4 +- .../_legacy_conda_imports/instructions.py | 38 ------------ conda_build/_legacy_conda_imports/plan.py | 59 ++++++++----------- 3 files changed, 27 insertions(+), 74 deletions(-) delete mode 100644 conda_build/_legacy_conda_imports/instructions.py diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 5bf0e4f7f4..b7652111b4 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -1,9 +1,7 @@ from .conda_imports import get_index -from .instructions import ( +from .plan import ( LINK, PREFIX, -) -from .plan import ( display_actions, execute_actions, install_actions, diff --git a/conda_build/_legacy_conda_imports/instructions.py b/conda_build/_legacy_conda_imports/instructions.py deleted file mode 100644 index ddc5b2b387..0000000000 --- a/conda_build/_legacy_conda_imports/instructions.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (C) 2012 Anaconda, Inc -# SPDX-License-Identifier: BSD-3-Clause -"""Define the instruction set (constants) for conda operations.""" -from logging import getLogger - -from .conda_imports import ProgressiveFetchExtract, UnlinkLinkTransaction - -log = getLogger(__name__) - -# op codes -PREFIX = "PREFIX" -LINK = "LINK" -UNLINKLINKTRANSACTION = "UNLINKLINKTRANSACTION" -PROGRESSIVEFETCHEXTRACT = "PROGRESSIVEFETCHEXTRACT" - - -def PREFIX_CMD(prefix): - pass - - -def PROGRESSIVEFETCHEXTRACT_CMD(progressive_fetch_extract): # pragma: no cover - assert isinstance(progressive_fetch_extract, ProgressiveFetchExtract) - progressive_fetch_extract.execute() - - -def UNLINKLINKTRANSACTION_CMD(arg): # pragma: no cover - unlink_link_transaction = arg - assert isinstance(unlink_link_transaction, UnlinkLinkTransaction) - unlink_link_transaction.execute() - - -# Map instruction to command (a python function) -commands = { - PREFIX: PREFIX_CMD, - LINK: None, - UNLINKLINKTRANSACTION: UNLINKLINKTRANSACTION_CMD, - PROGRESSIVEFETCHEXTRACT: PROGRESSIVEFETCHEXTRACT_CMD, -} diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 8c4b05a47e..6ebef29912 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -33,13 +33,10 @@ stack_context_default, ) -from .instructions import ( - LINK, - PREFIX, - PROGRESSIVEFETCHEXTRACT, - UNLINKLINKTRANSACTION, - commands, -) +PREFIX = "PREFIX" +LINK = "LINK" +UNLINKLINKTRANSACTION = "UNLINKLINKTRANSACTION" +PROGRESSIVEFETCHEXTRACT = "PROGRESSIVEFETCHEXTRACT" log = getLogger(__name__) @@ -117,7 +114,24 @@ def format(s, pkg): def execute_actions(actions, verbose=False): # pragma: no cover plan = _plan_from_actions(actions) - execute_instructions(plan, verbose) + log.debug("executing plan %s", plan) + + for instruction, arg in plan: + log.debug(" %s(%r)", instruction, arg) + + if instruction in (PREFIX, LINK): + continue + + if instruction == PROGRESSIVEFETCHEXTRACT: + progressive_fetch_extract = arg + assert isinstance(progressive_fetch_extract, ProgressiveFetchExtract) + progressive_fetch_extract.execute() + elif instruction == UNLINKLINKTRANSACTION: + unlink_link_transaction = arg + assert isinstance(unlink_link_transaction, UnlinkLinkTransaction) + unlink_link_transaction.execute() + else: + raise KeyError(instruction) def _plan_from_actions(actions): # pragma: no cover @@ -246,29 +260,8 @@ def install_actions( solver._index = index.copy() txn = solver.solve_for_transaction(prune=prune, ignore_pinned=not pinned) prefix_setup = txn.prefix_setups[prefix] - actions = get_blank_actions(prefix) - actions[LINK].extend(prec for prec in prefix_setup.link_precs) + actions = { + PREFIX: prefix, + LINK: [prec for prec in prefix_setup.link_precs], + } return actions - - -def get_blank_actions(prefix): # pragma: no cover - actions = defaultdict(list) - actions[PREFIX] = prefix - return actions - - -def execute_instructions(plan, verbose=False): - """Execute the instructions in the plan - - :param plan: A list of (instruction, arg) tuples - :param verbose: verbose output - """ - log.debug("executing plan %s", plan) - - for instruction, arg in plan: - log.debug(" %s(%r)", instruction, arg) - - cmd = commands[instruction] - - if callable(cmd): - cmd(arg) From 240baf35d3b934d73df5aa4b9816fb0f543b9b1a Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 04:28:01 +0100 Subject: [PATCH 34/61] Simplify _plan_from_actions Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/plan.py | 75 ++++++----------------- 1 file changed, 18 insertions(+), 57 deletions(-) diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 6ebef29912..3c9b7dc98f 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -139,70 +139,31 @@ def _plan_from_actions(actions): # pragma: no cover prefix = actions[PREFIX] plan = [(PREFIX, "%s" % prefix)] - unlink_link_transaction = actions.get(UNLINKLINKTRANSACTION) - if unlink_link_transaction: - raise RuntimeError() - # progressive_fetch_extract = actions.get(PROGRESSIVEFETCHEXTRACT) - # if progressive_fetch_extract: - # plan.append((PROGRESSIVEFETCHEXTRACT, progressive_fetch_extract)) - # plan.append((UNLINKLINKTRANSACTION, unlink_link_transaction)) - # return plan - - log.debug(f"Adding plans for operations: {[LINK]}") if LINK not in actions: - log.trace(f"action {LINK} not in actions") - elif not actions[LINK]: - log.trace(f"action {LINK} has None value") - else: - for arg in actions[LINK]: - log.debug(f"appending value {arg} for action {LINK}") - plan.append((LINK, arg)) + log.debug(f"action {LINK} not in actions") + return plan - plan = _inject_UNLINKLINKTRANSACTION(plan, prefix) + link_precs = actions[LINK] + if not link_precs: + log.debug(f"action {LINK} has None value") + return plan - return plan - - -def _inject_UNLINKLINKTRANSACTION(plan, prefix): # pragma: no cover - # this is only used for conda-build at this point - first_unlink_link_idx = next( - (q for q, p in enumerate(plan) if p[0] in (LINK,)), -1 - ) - if first_unlink_link_idx >= 0: - link_precs = tuple(prec for action, prec in plan if action == LINK) - link_precs = _handle_menuinst(link_precs) - - pfe = ProgressiveFetchExtract(link_precs) - pfe.prepare() - - stp = PrefixSetup(prefix, (), link_precs, (), [], ()) - plan.insert( - first_unlink_link_idx, (UNLINKLINKTRANSACTION, UnlinkLinkTransaction(stp)) + if on_win: + # Always link menuinst first/last on windows in case a subsequent + # package tries to import it to create/remove a shortcut + link_precs = ( + [p for p in link_precs if p.name == "menuinst"] + + [p for p in link_precs if p.name != "menuinst"] ) - plan.insert(first_unlink_link_idx, (PROGRESSIVEFETCHEXTRACT, pfe)) - return plan + pfe = ProgressiveFetchExtract(link_precs) + pfe.prepare() + plan.append((PROGRESSIVEFETCHEXTRACT, pfe)) + stp = PrefixSetup(prefix, (), link_precs, (), [], ()) + plan.append((UNLINKLINKTRANSACTION, UnlinkLinkTransaction(stp))) -def _handle_menuinst(link_precs): # pragma: no cover - if not on_win: - return link_precs - - # Always link menuinst first/last on windows in case a subsequent - # package tries to import it to create/remove a shortcut - - # link - menuinst_idx = next( - (q for q, d in enumerate(link_precs) if d.name == "menuinst"), None - ) - if menuinst_idx is not None: - link_precs = ( - *link_precs[menuinst_idx : menuinst_idx + 1], - *link_precs[:menuinst_idx], - *link_precs[menuinst_idx + 1 :], - ) - - return link_precs + return plan def install_actions( From f410de42859b68be42dae8ad0153722c8abb264d Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 04:38:44 +0100 Subject: [PATCH 35/61] Collapse _plan_from_actions into execute_actions Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/plan.py | 43 ++++++----------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 3c9b7dc98f..b601e0c450 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -35,8 +35,6 @@ PREFIX = "PREFIX" LINK = "LINK" -UNLINKLINKTRANSACTION = "UNLINKLINKTRANSACTION" -PROGRESSIVEFETCHEXTRACT = "PROGRESSIVEFETCHEXTRACT" log = getLogger(__name__) @@ -112,41 +110,18 @@ def format(s, pkg): # ---------------------------- Backwards compat for conda-build -------------------------- -def execute_actions(actions, verbose=False): # pragma: no cover - plan = _plan_from_actions(actions) - log.debug("executing plan %s", plan) - - for instruction, arg in plan: - log.debug(" %s(%r)", instruction, arg) - - if instruction in (PREFIX, LINK): - continue - - if instruction == PROGRESSIVEFETCHEXTRACT: - progressive_fetch_extract = arg - assert isinstance(progressive_fetch_extract, ProgressiveFetchExtract) - progressive_fetch_extract.execute() - elif instruction == UNLINKLINKTRANSACTION: - unlink_link_transaction = arg - assert isinstance(unlink_link_transaction, UnlinkLinkTransaction) - unlink_link_transaction.execute() - else: - raise KeyError(instruction) - - -def _plan_from_actions(actions): # pragma: no cover +def execute_actions(actions, verbose=False): assert PREFIX in actions and actions[PREFIX] prefix = actions[PREFIX] - plan = [(PREFIX, "%s" % prefix)] if LINK not in actions: log.debug(f"action {LINK} not in actions") - return plan + return link_precs = actions[LINK] if not link_precs: log.debug(f"action {LINK} has None value") - return plan + return if on_win: # Always link menuinst first/last on windows in case a subsequent @@ -156,14 +131,16 @@ def _plan_from_actions(actions): # pragma: no cover [p for p in link_precs if p.name != "menuinst"] ) - pfe = ProgressiveFetchExtract(link_precs) - pfe.prepare() - plan.append((PROGRESSIVEFETCHEXTRACT, pfe)) + progressive_fetch_extract = ProgressiveFetchExtract(link_precs) + progressive_fetch_extract.prepare() stp = PrefixSetup(prefix, (), link_precs, (), [], ()) - plan.append((UNLINKLINKTRANSACTION, UnlinkLinkTransaction(stp))) + unlink_link_transaction = UnlinkLinkTransaction(stp) - return plan + log.debug(" %s(%r)", "PROGRESSIVEFETCHEXTRACT", progressive_fetch_extract) + progressive_fetch_extract.execute() + log.debug(" %s(%r)", "UNLINKLINKTRANSACTION", unlink_link_transaction) + unlink_link_transaction.execute() def install_actions( From 5cabb3471ceb83030dbb90714ca83548aca28015 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 04:52:06 +0100 Subject: [PATCH 36/61] Remove unused args for execute/install_actions Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/plan.py | 53 ++++++----------------- conda_build/environ.py | 2 +- 2 files changed, 14 insertions(+), 41 deletions(-) diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index b601e0c450..220270fd7e 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -12,7 +12,6 @@ from collections import defaultdict from logging import getLogger -from os.path import basename, isdir from .conda_imports import ( DEFAULTS_CHANNEL_NAME, @@ -38,8 +37,6 @@ log = getLogger(__name__) -# TODO: Remove conda/plan.py. This module should be almost completely deprecated now. - def display_actions(actions): prefix = actions.get(PREFIX) @@ -107,10 +104,7 @@ def format(s, pkg): print() -# ---------------------------- Backwards compat for conda-build -------------------------- - - -def execute_actions(actions, verbose=False): +def execute_actions(actions): assert PREFIX in actions and actions[PREFIX] prefix = actions[PREFIX] @@ -143,21 +137,7 @@ def execute_actions(actions, verbose=False): unlink_link_transaction.execute() -def install_actions( - prefix, - index, - specs, - force=False, - only_names=None, - always_copy=False, - pinned=True, - update_deps=True, - prune=False, - channel_priority_map=None, - is_update=False, - minimal_hint=False, -): # pragma: no cover - # this is for conda-build +def install_actions(prefix, index, specs): with env_vars( { "CONDA_ALLOW_NON_CHANNEL_URLS": "true", @@ -165,25 +145,18 @@ def install_actions( }, stack_callback=stack_context_default, ): - if channel_priority_map: - channel_names = IndexedSet( - Channel(url).canonical_name for url in channel_priority_map + # a hack since in conda-build we don't track channel_priority_map + if LAST_CHANNEL_URLS: + channel_priority_map = prioritize_channels(LAST_CHANNEL_URLS) + channels = IndexedSet(Channel(url) for url in channel_priority_map) + subdirs = ( + IndexedSet( + subdir for subdir in (c.subdir for c in channels) if subdir + ) + or context.subdirs ) - channels = IndexedSet(Channel(cn) for cn in channel_names) - subdirs = IndexedSet(basename(url) for url in channel_priority_map) else: - # a hack for when conda-build calls this function without giving channel_priority_map - if LAST_CHANNEL_URLS: - channel_priority_map = prioritize_channels(LAST_CHANNEL_URLS) - channels = IndexedSet(Channel(url) for url in channel_priority_map) - subdirs = ( - IndexedSet( - subdir for subdir in (c.subdir for c in channels) if subdir - ) - or context.subdirs - ) - else: - channels = subdirs = None + channels = subdirs = None specs = tuple(MatchSpec(spec) for spec in specs) @@ -196,7 +169,7 @@ def install_actions( # package) => Copy index (just outer container, not deep copy) # to conserve it. solver._index = index.copy() - txn = solver.solve_for_transaction(prune=prune, ignore_pinned=not pinned) + txn = solver.solve_for_transaction(prune=False, ignore_pinned=False) prefix_setup = txn.prefix_setups[prefix] actions = { PREFIX: prefix, diff --git a/conda_build/environ.py b/conda_build/environ.py index e48c6175a1..28d394bc2e 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -904,7 +904,7 @@ def get_install_actions( with utils.LoggingContext(conda_log_level): with capture(): try: - actions = install_actions(prefix, index, specs, force=True) + actions = install_actions(prefix, index, specs) except (NoPackagesFoundError, UnsatisfiableError) as exc: raise DependencyNeedsBuildingError(exc, subdir=subdir) except ( From febb326dfc9a8b92d9e82ef8ed36e426d26af90f Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 05:03:14 +0100 Subject: [PATCH 37/61] Remove clean_pkg_cache and related dist functions Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 23 ++------- conda_build/environ.py | 59 ----------------------- 2 files changed, 3 insertions(+), 79 deletions(-) diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index a6944cf20e..4c6e24be12 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -105,26 +105,9 @@ def _as_dict_from_string(string, channel_override=_not_set): ) -def package_ref_from_dist_string(dist_string): - dist_kwargs = _as_dict_from_string(dist_string) - return PackageRecord( - channel=dist_kwargs["channel"], - name=dist_kwargs["name"], - version=dist_kwargs["version"], - build=dist_kwargs["build"], - build_number=dist_kwargs["build_number"], - ) - - -def dist_string_contains(containing_dist_string, contained_dist_string): - contained_dist_string = _strip_extension(contained_dist_string) - return contained_dist_string in containing_dist_string - - -def dist_string_from_package_record(package_record, channel=None): - if channel is None: - channel = package_record.channel.canonical_name +def dist_string_from_package_record(package_record): dist_kwargs = _as_dict_from_string( - package_record.fn, channel_override=channel + package_record.fn, + channel_override=package_record.channel.canonical_name, ) return dist_kwargs["dist_name"] diff --git a/conda_build/environ.py b/conda_build/environ.py index 28d394bc2e..216dc70e6e 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -1221,65 +1221,6 @@ def remove_existing_packages(dirs, fns, config): utils.rm_rf(entry) -@deprecated("3.28.0", "24.1.0") -def clean_pkg_cache(dist_string, config): - from ._legacy_conda_imports.dist import ( - dist_string_contains, - dist_string_from_package_record, - package_ref_from_dist_string, - ) - locks = [] - - conda_log_level = logging.WARN - if config.debug: - conda_log_level = logging.DEBUG - - with utils.LoggingContext(conda_log_level): - locks = get_pkg_dirs_locks([config.bldpkgs_dir] + pkgs_dirs, config) - with utils.try_acquire_locks(locks, timeout=config.timeout): - # NOTE: The following out-commented execute_plan was defunct - # (RM_* were no-ops). - # - # rmplan = [ - # f"{RM_EXTRACTED} {dist} local::{dist}", - # f"{RM_FETCHED} {dist} local::{dist}", - # ] - # execute_plan(rmplan) - - # Conda does not seem to do a complete cleanup sometimes. This is supplemental. - # Conda's cleanup is still necessary - it keeps track of its own in-memory - # list of downloaded things. - package_cache_data = PackageCacheData.first_writable() - package_records = [ - package_ref_from_dist_string(pkg_id) - for pkg_id in [dist_string, "local::" + dist_string] - ] - for folder in pkgs_dirs: - if ( - os.path.exists(os.path.join(folder, dist_string)) - or os.path.exists(os.path.join(folder, dist_string + ".tar.bz2")) - or any( - bool(PackageCacheData.first_writable().get(package_record, None)) - for package_record in package_records - ) - ): - log = utils.get_logger(__name__) - log.debug( - "Conda caching error: %s package remains in cache after removal", - dist_string, - ) - log.debug("manually removing to compensate") - for cache_pkg_id in package_cache_data.values(): - cache_pkg_id_dist_string = dist_string_from_package_record(cache_pkg_id) - if dist_string_contains(cache_pkg_id_dist_string, dist_string): - if bool(package_cache_data.get(cache_pkg_id, None)): - package_cache_data.remove(cache_pkg_id) - - # Note that this call acquires the relevant locks, so this must be called - # outside the lock context above. - remove_existing_packages(pkgs_dirs, [dist_string], config) - - def get_pinned_deps(m, section): with TemporaryDirectory(prefix="_") as tmpdir: actions = get_install_actions( From f81c98324b6100365126e2c9241af393dc724d8b Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 05:12:10 +0100 Subject: [PATCH 38/61] Remove all unused code from dist string handling --- .../_legacy_conda_imports/conda_imports.py | 4 - conda_build/_legacy_conda_imports/dist.py | 103 ++---------------- 2 files changed, 8 insertions(+), 99 deletions(-) diff --git a/conda_build/_legacy_conda_imports/conda_imports.py b/conda_build/_legacy_conda_imports/conda_imports.py index 7df4d71d91..a9203a000a 100644 --- a/conda_build/_legacy_conda_imports/conda_imports.py +++ b/conda_build/_legacy_conda_imports/conda_imports.py @@ -5,22 +5,18 @@ from conda.exports import ( Channel, - CondaError, MatchSpec, PackageRecord, ProgressiveFetchExtract, on_win, - normalized_version, ) from conda.base.constants import ( - CONDA_PACKAGE_EXTENSIONS, DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL, ) from conda.base.context import context, stack_context_default from conda.common.io import env_vars -from conda.common.url import is_url from conda.core.index import LAST_CHANNEL_URLS, get_index from conda.core.link import PrefixSetup, UnlinkLinkTransaction from conda.core.prefix_data import PrefixData diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py index 4c6e24be12..058b52199c 100644 --- a/conda_build/_legacy_conda_imports/dist.py +++ b/conda_build/_legacy_conda_imports/dist.py @@ -2,78 +2,12 @@ # SPDX-License-Identifier: BSD-3-Clause """(Legacy) Low-level implementation of a Channel.""" import re -from typing import NamedTuple -from .conda_imports import ( - CONDA_PACKAGE_EXTENSIONS, - UNKNOWN_CHANNEL, - CondaError, - PackageRecord, - is_url, -) +from conda.base.constants import CONDA_PACKAGE_EXTENSIONS -class DistDetails(NamedTuple): - name: str - version: str - build: str - build_number: str - dist_name: str - fmt: str - - -def _strip_extension(original_dist): - for ext in CONDA_PACKAGE_EXTENSIONS: - if original_dist.endswith(ext): - original_dist = original_dist[: -len(ext)] - return original_dist - - -def _split_extension(original_dist): - stripped = _strip_extension(original_dist) - return stripped, original_dist[len(stripped) :] - - -def _parse_dist_name(string): - original_string = string - try: - no_fmt_string, fmt = _split_extension(string) - - # remove any directory or channel information - if "::" in no_fmt_string: - dist_name = no_fmt_string.rsplit("::", 1)[-1] - else: - dist_name = no_fmt_string.rsplit("/", 1)[-1] - - parts = dist_name.rsplit("-", 2) - - name = parts[0] - version = parts[1] - build = parts[2] if len(parts) >= 3 else "" - build_number_as_string = "".join( - filter( - lambda x: x.isdigit(), - (build.rsplit("_")[-1] if build else "0"), - ) - ) - build_number = int(build_number_as_string) if build_number_as_string else 0 - - return DistDetails( - name, version, build, build_number, dist_name, fmt - ) - - except: - raise CondaError( - "dist_name is not a valid conda package: %s" % original_string - ) - - -_not_set = object() - - -def _as_dict_from_string(string, channel_override=_not_set): - if is_url(string) and channel_override == _not_set: - raise NotImplementedError() +def dist_string_from_package_record(package_record): + string = package_record.fn if string.endswith("@"): raise NotImplementedError() @@ -85,29 +19,8 @@ def _as_dict_from_string(string, channel_override=_not_set): ) channel, original_dist, w_f_d = re.search(REGEX_STR, string).groups() - original_dist, fmt = _split_extension(original_dist) - - if channel_override != _not_set: - channel = channel_override - if not channel: - channel = UNKNOWN_CHANNEL - - # enforce dist format - dist_details = _parse_dist_name(original_dist) - return dict( - channel=channel, - name=dist_details.name, - version=dist_details.version, - build=dist_details.build, - build_number=dist_details.build_number, - dist_name=original_dist, - fmt=fmt, - ) - - -def dist_string_from_package_record(package_record): - dist_kwargs = _as_dict_from_string( - package_record.fn, - channel_override=package_record.channel.canonical_name, - ) - return dist_kwargs["dist_name"] + stripped = original_dist + for ext in CONDA_PACKAGE_EXTENSIONS: + if stripped.endswith(ext): + stripped = stripped[: -len(ext)] + return stripped From b9040d0bb3e5d6f356d5b0afaff7d37eb1f41a21 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 05:17:53 +0100 Subject: [PATCH 39/61] Move dist_string_from_package_record to .utils Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/dist.py | 26 ----------------------- conda_build/environ.py | 2 +- conda_build/render.py | 7 ++++-- conda_build/utils.py | 20 +++++++++++++++++ 4 files changed, 26 insertions(+), 29 deletions(-) delete mode 100644 conda_build/_legacy_conda_imports/dist.py diff --git a/conda_build/_legacy_conda_imports/dist.py b/conda_build/_legacy_conda_imports/dist.py deleted file mode 100644 index 058b52199c..0000000000 --- a/conda_build/_legacy_conda_imports/dist.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (C) 2012 Anaconda, Inc -# SPDX-License-Identifier: BSD-3-Clause -"""(Legacy) Low-level implementation of a Channel.""" -import re - -from conda.base.constants import CONDA_PACKAGE_EXTENSIONS - - -def dist_string_from_package_record(package_record): - string = package_record.fn - - if string.endswith("@"): - raise NotImplementedError() - - REGEX_STR = ( - r"(?:([^\s\[\]]+)::)?" # optional channel - r"([^\s\[\]]+)" # 3.x dist - r"(?:\[([a-zA-Z0-9_-]+)\])?" # with_features_depends - ) - channel, original_dist, w_f_d = re.search(REGEX_STR, string).groups() - - stripped = original_dist - for ext in CONDA_PACKAGE_EXTENSIONS: - if stripped.endswith(ext): - stripped = stripped[: -len(ext)] - return stripped diff --git a/conda_build/environ.py b/conda_build/environ.py index 216dc70e6e..3e0561c7a0 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -35,7 +35,6 @@ reset_context, root_dir, ) -from ._legacy_conda_imports.dist import dist_string_from_package_record from .deprecations import deprecated from .exceptions import BuildLockError, DependencyNeedsBuildingError from .features import feature_list @@ -43,6 +42,7 @@ from .metadata import MetaData from .os_utils import external from .utils import ( + dist_string_from_package_record, ensure_list, env_var, on_mac, diff --git a/conda_build/render.py b/conda_build/render.py index 945de5268d..e47b9e43dc 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -35,11 +35,14 @@ pkgs_dirs, specs_from_url, ) -from ._legacy_conda_imports.dist import dist_string_from_package_record from .exceptions import DependencyNeedsBuildingError from .index import get_build_index from .metadata import MetaData, combine_top_level_metadata_with_output -from .utils import CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2 +from .utils import ( + CONDA_PACKAGE_EXTENSION_V1, + CONDA_PACKAGE_EXTENSION_V2, + dist_string_from_package_record +) from .variants import ( filter_by_key_value, get_package_variants, diff --git a/conda_build/utils.py b/conda_build/utils.py index 7b8772d8a1..3625ab5d2f 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -2180,3 +2180,23 @@ def samefile(path1: Path, path2: Path) -> bool: # FileNotFoundError: path doesn't exist # PermissionError: don't have permissions to read path return path1 == path2 + + +def dist_string_from_package_record(package_record): + string = package_record.fn + + if string.endswith("@"): + raise NotImplementedError() + + REGEX_STR = ( + r"(?:([^\s\[\]]+)::)?" # optional channel + r"([^\s\[\]]+)" # 3.x dist + r"(?:\[([a-zA-Z0-9_-]+)\])?" # with_features_depends + ) + channel, original_dist, w_f_d = re.search(REGEX_STR, string).groups() + + stripped = original_dist + for ext in CONDA_PACKAGE_EXTENSIONS: + if stripped.endswith(ext): + stripped = stripped[: -len(ext)] + return stripped From 329f4a1348c22e0739c7044e0ada80e094d0b5f6 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 05:23:11 +0100 Subject: [PATCH 40/61] Use conda.core.index.get_index in .conda_interface Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 1 - conda_build/conda_interface.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index b7652111b4..0f85b04d78 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -1,4 +1,3 @@ -from .conda_imports import get_index from .plan import ( LINK, PREFIX, diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index d1efa993cc..aa130ab466 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -16,6 +16,7 @@ ) from conda.base.context import context, determine_target_prefix, reset_context from conda.base.context import non_x86_machines as non_x86_linux_machines # noqa: F401 +from conda.core.index import get_index from conda.core.package_cache import ProgressiveFetchExtract # noqa: F401 from conda.core.package_cache_data import PackageCacheData # noqa: F401 from conda.exceptions import ( # noqa: F401 @@ -160,6 +161,5 @@ def which_prefix(path: str | os.PathLike | Path) -> Path: PREFIX, display_actions, execute_actions, - get_index, install_actions, ) From c36987863a37b1a2a1eb692b62419cf1ef65130b Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 05:28:53 +0100 Subject: [PATCH 41/61] Rename PREFIX, LINK to PREFIX_ACTION, LINK_ACTION Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 4 ++-- conda_build/_legacy_conda_imports/plan.py | 24 +++++++++---------- conda_build/conda_interface.py | 4 ++-- conda_build/environ.py | 17 ++++++------- conda_build/render.py | 10 ++++---- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py index 0f85b04d78..df5eb1d92b 100644 --- a/conda_build/_legacy_conda_imports/__init__.py +++ b/conda_build/_legacy_conda_imports/__init__.py @@ -1,6 +1,6 @@ from .plan import ( - LINK, - PREFIX, + LINK_ACTION, + PREFIX_ACTION, display_actions, execute_actions, install_actions, diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/_legacy_conda_imports/plan.py index 220270fd7e..d9dec89992 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/_legacy_conda_imports/plan.py @@ -32,14 +32,14 @@ stack_context_default, ) -PREFIX = "PREFIX" -LINK = "LINK" +PREFIX_ACTION = "PREFIX" +LINK_ACTION = "LINK" log = getLogger(__name__) def display_actions(actions): - prefix = actions.get(PREFIX) + prefix = actions.get(PREFIX_ACTION) builder = ["", "## Package Plan ##\n"] if prefix: builder.append(" environment location: %s" % prefix) @@ -68,7 +68,7 @@ def channel_filt(s): features = defaultdict(lambda: "") channels = defaultdict(lambda: "") - for prec in actions.get(LINK, []): + for prec in actions.get(LINK_ACTION, []): assert isinstance(prec, PackageRecord) pkg = prec["name"] channels[pkg] = channel_filt(channel_str(prec)) @@ -105,16 +105,16 @@ def format(s, pkg): def execute_actions(actions): - assert PREFIX in actions and actions[PREFIX] - prefix = actions[PREFIX] + assert PREFIX_ACTION in actions and actions[PREFIX_ACTION] + prefix = actions[PREFIX_ACTION] - if LINK not in actions: - log.debug(f"action {LINK} not in actions") + if LINK_ACTION not in actions: + log.debug(f"action {LINK_ACTION} not in actions") return - link_precs = actions[LINK] + link_precs = actions[LINK_ACTION] if not link_precs: - log.debug(f"action {LINK} has None value") + log.debug(f"action {LINK_ACTION} has None value") return if on_win: @@ -172,7 +172,7 @@ def install_actions(prefix, index, specs): txn = solver.solve_for_transaction(prune=False, ignore_pinned=False) prefix_setup = txn.prefix_setups[prefix] actions = { - PREFIX: prefix, - LINK: [prec for prec in prefix_setup.link_precs], + PREFIX_ACTION: prefix, + LINK_ACTION: [prec for prec in prefix_setup.link_precs], } return actions diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index aa130ab466..2ec9097fbf 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -157,8 +157,8 @@ def which_prefix(path: str | os.PathLike | Path) -> Path: env_path_backup_var_exists = os.environ.get("CONDA_PATH_BACKUP", None) from ._legacy_conda_imports import ( - LINK, - PREFIX, + LINK_ACTION, + PREFIX_ACTION, display_actions, execute_actions, install_actions, diff --git a/conda_build/environ.py b/conda_build/environ.py index 3e0561c7a0..dfa038917f 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -16,8 +16,8 @@ from . import utils from .conda_interface import ( - LINK, - PREFIX, + LINK_ACTION, + PREFIX_ACTION, CondaError, LinkError, LockError, @@ -895,8 +895,8 @@ def get_install_actions( disable_pip, ) in cached_actions and last_index_ts >= index_ts: actions = cached_actions[(specs, env, subdir, channel_urls, disable_pip)].copy() - if PREFIX in actions: - actions[PREFIX] = prefix + if PREFIX_ACTION in actions: + actions[PREFIX_ACTION] = prefix elif specs: # this is hiding output like: # Fetching package metadata ........... @@ -978,8 +978,8 @@ def get_install_actions( if not any( re.match(r"^%s(?:$|[\s=].*)" % pkg, str(dep)) for dep in specs ): - actions[LINK] = [ - spec for spec in actions[LINK] if spec.name != pkg + actions[LINK_ACTION] = [ + spec for spec in actions[LINK_ACTION] if spec.name != pkg ] utils.trim_empty_keys(actions) cached_actions[(specs, env, subdir, channel_urls, disable_pip)] = actions.copy() @@ -1101,7 +1101,7 @@ def create_env( # Set this here and use to create environ # Setting this here is important because we use it below (symlink) prefix = config.host_prefix if host else config.build_prefix - actions[PREFIX] = prefix + actions[PREFIX_ACTION] = prefix create_env( prefix, @@ -1239,6 +1239,7 @@ def get_pinned_deps(m, section): channel_urls=tuple(m.config.channel_urls), ) runtime_deps = [ - " ".join(dist_string_from_package_record(link).rsplit("-", 2)) for link in actions.get(LINK, []) + " ".join(dist_string_from_package_record(link).rsplit("-", 2)) + for link in actions.get(LINK_ACTION, []) ] return runtime_deps diff --git a/conda_build/render.py b/conda_build/render.py index e47b9e43dc..3c176576a3 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -28,7 +28,7 @@ from . import environ, exceptions, source, utils from .conda_interface import ( - LINK, + LINK_ACTION, ProgressiveFetchExtract, TemporaryDirectory, UnsatisfiableError, @@ -90,10 +90,10 @@ def bldpkg_path(m): def actions_to_pins(actions): - if LINK in actions: + if LINK_ACTION in actions: return [ " ".join(dist_string_from_package_record(spec).split()[0].rsplit("-", 2)) - for spec in actions[LINK] + for spec in actions[LINK_ACTION] ] return [] @@ -357,7 +357,7 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files pkg_files = {} - packages = actions.get(LINK, []) + packages = actions.get(LINK_ACTION, []) selected_packages = set() if package_subset: for pkg in package_subset: @@ -401,7 +401,7 @@ def get_upstream_pins(m: MetaData, actions, env): downstream dependency specs. Return these additional specs.""" env_specs = m.get_value(f"requirements/{env}", []) explicit_specs = [req.split(" ")[0] for req in env_specs] if env_specs else [] - linked_packages = actions.get(LINK, []) + linked_packages = actions.get(LINK_ACTION, []) linked_packages = [pkg for pkg in linked_packages if pkg.name in explicit_specs] ignore_pkgs_list = utils.ensure_list(m.get_value("build/ignore_run_exports_from")) From 50df3742a49582108e3be3a10be85a11017c5c6d Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 05:39:32 +0100 Subject: [PATCH 42/61] Move plan.py *_actions to .environ_actions Signed-off-by: Marcel Bargull --- conda_build/_legacy_conda_imports/__init__.py | 7 ------ .../_legacy_conda_imports/conda_imports.py | 23 ----------------- conda_build/conda_interface.py | 8 ------ conda_build/environ.py | 12 +++++---- .../plan.py => environ_actions.py} | 25 +++++++++++-------- conda_build/render.py | 2 +- 6 files changed, 23 insertions(+), 54 deletions(-) delete mode 100644 conda_build/_legacy_conda_imports/__init__.py delete mode 100644 conda_build/_legacy_conda_imports/conda_imports.py rename conda_build/{_legacy_conda_imports/plan.py => environ_actions.py} (91%) diff --git a/conda_build/_legacy_conda_imports/__init__.py b/conda_build/_legacy_conda_imports/__init__.py deleted file mode 100644 index df5eb1d92b..0000000000 --- a/conda_build/_legacy_conda_imports/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .plan import ( - LINK_ACTION, - PREFIX_ACTION, - display_actions, - execute_actions, - install_actions, -) diff --git a/conda_build/_legacy_conda_imports/conda_imports.py b/conda_build/_legacy_conda_imports/conda_imports.py deleted file mode 100644 index a9203a000a..0000000000 --- a/conda_build/_legacy_conda_imports/conda_imports.py +++ /dev/null @@ -1,23 +0,0 @@ -try: - from boltons.setutils import IndexedSet -except ImportError: # pragma: no cover - from conda._vendor.boltons.setutils import IndexedSet - -from conda.exports import ( - Channel, - MatchSpec, - PackageRecord, - ProgressiveFetchExtract, - on_win, -) - -from conda.base.constants import ( - DEFAULTS_CHANNEL_NAME, - UNKNOWN_CHANNEL, -) -from conda.base.context import context, stack_context_default -from conda.common.io import env_vars -from conda.core.index import LAST_CHANNEL_URLS, get_index -from conda.core.link import PrefixSetup, UnlinkLinkTransaction -from conda.core.prefix_data import PrefixData -from conda.models.channel import prioritize_channels diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 2ec9097fbf..8a12a1abe8 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -155,11 +155,3 @@ def which_prefix(path: str | os.PathLike | Path) -> Path: # When deactivating envs (e.g. switching from root to build/test) this env var is used, # except the PR that removed this has been reverted (for now) and Windows doesn't need it. env_path_backup_var_exists = os.environ.get("CONDA_PATH_BACKUP", None) - -from ._legacy_conda_imports import ( - LINK_ACTION, - PREFIX_ACTION, - display_actions, - execute_actions, - install_actions, -) diff --git a/conda_build/environ.py b/conda_build/environ.py index dfa038917f..3d2d5d6885 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -16,8 +16,6 @@ from . import utils from .conda_interface import ( - LINK_ACTION, - PREFIX_ACTION, CondaError, LinkError, LockError, @@ -27,15 +25,19 @@ TemporaryDirectory, UnsatisfiableError, create_default_packages, - display_actions, - execute_actions, get_version_from_git_tag, - install_actions, pkgs_dirs, reset_context, root_dir, ) from .deprecations import deprecated +from .environ_actions import ( + LINK_ACTION, + PREFIX_ACTION, + display_actions, + execute_actions, + install_actions, +) from .exceptions import BuildLockError, DependencyNeedsBuildingError from .features import feature_list from .index import get_build_index diff --git a/conda_build/_legacy_conda_imports/plan.py b/conda_build/environ_actions.py similarity index 91% rename from conda_build/_legacy_conda_imports/plan.py rename to conda_build/environ_actions.py index d9dec89992..806a4020ec 100644 --- a/conda_build/_legacy_conda_imports/plan.py +++ b/conda_build/environ_actions.py @@ -13,25 +13,30 @@ from collections import defaultdict from logging import getLogger -from .conda_imports import ( +try: + from boltons.setutils import IndexedSet +except ImportError: # pragma: no cover + from conda._vendor.boltons.setutils import IndexedSet + +from conda.base.constants import ( DEFAULTS_CHANNEL_NAME, - LAST_CHANNEL_URLS, UNKNOWN_CHANNEL, +) +from conda.base.context import context, stack_context_default +from conda.common.io import env_vars +from conda.core.index import LAST_CHANNEL_URLS +from conda.core.link import PrefixSetup, UnlinkLinkTransaction +from conda.core.prefix_data import PrefixData +from conda.exports import ( Channel, - IndexedSet, MatchSpec, PackageRecord, - PrefixData, - PrefixSetup, ProgressiveFetchExtract, - UnlinkLinkTransaction, - context, - env_vars, on_win, - prioritize_channels, - stack_context_default, ) +from conda.models.channel import prioritize_channels + PREFIX_ACTION = "PREFIX" LINK_ACTION = "LINK" diff --git a/conda_build/render.py b/conda_build/render.py index 3c176576a3..c9a7ddef05 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -28,13 +28,13 @@ from . import environ, exceptions, source, utils from .conda_interface import ( - LINK_ACTION, ProgressiveFetchExtract, TemporaryDirectory, UnsatisfiableError, pkgs_dirs, specs_from_url, ) +from .environ_actions import LINK_ACTION from .exceptions import DependencyNeedsBuildingError from .index import get_build_index from .metadata import MetaData, combine_top_level_metadata_with_output From ac6d9de56deeed0f953e338a789149ee4c99ccaa Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 05:57:29 +0100 Subject: [PATCH 43/61] Linting fixes Signed-off-by: Marcel Bargull --- conda_build/conda_interface.py | 3 +-- conda_build/environ.py | 2 -- conda_build/environ_actions.py | 19 +++++++++---------- conda_build/inspect_pkg.py | 1 - conda_build/post.py | 2 +- conda_build/render.py | 12 ++++++++---- conda_build/utils.py | 2 -- 7 files changed, 19 insertions(+), 22 deletions(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 8a12a1abe8..6e2412947b 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations - import configparser # noqa: F401 import os from functools import partial @@ -16,7 +15,7 @@ ) from conda.base.context import context, determine_target_prefix, reset_context from conda.base.context import non_x86_machines as non_x86_linux_machines # noqa: F401 -from conda.core.index import get_index +from conda.core.index import get_index # noqa: F401 from conda.core.package_cache import ProgressiveFetchExtract # noqa: F401 from conda.core.package_cache_data import PackageCacheData # noqa: F401 from conda.exceptions import ( # noqa: F401 diff --git a/conda_build/environ.py b/conda_build/environ.py index 3d2d5d6885..336f73c77b 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -20,7 +20,6 @@ LinkError, LockError, NoPackagesFoundError, - PackageCacheData, PaddingError, TemporaryDirectory, UnsatisfiableError, @@ -30,7 +29,6 @@ reset_context, root_dir, ) -from .deprecations import deprecated from .environ_actions import ( LINK_ACTION, PREFIX_ACTION, diff --git a/conda_build/environ_actions.py b/conda_build/environ_actions.py index 806a4020ec..fbfaacff41 100644 --- a/conda_build/environ_actions.py +++ b/conda_build/environ_actions.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Anaconda, Inc +# Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause """ Handle the planning of installs and their execution. @@ -34,7 +34,6 @@ ProgressiveFetchExtract, on_win, ) - from conda.models.channel import prioritize_channels PREFIX_ACTION = "PREFIX" @@ -99,7 +98,10 @@ def channel_filt(s): def format(s, pkg): return lead + s.format( - pkg=pkg + ":", vers=packages[pkg], channel=channels[pkg], features=features[pkg] + pkg=pkg + ":", + vers=packages[pkg], + channel=channels[pkg], + features=features[pkg], ) if packages: @@ -125,10 +127,9 @@ def execute_actions(actions): if on_win: # Always link menuinst first/last on windows in case a subsequent # package tries to import it to create/remove a shortcut - link_precs = ( - [p for p in link_precs if p.name == "menuinst"] + - [p for p in link_precs if p.name != "menuinst"] - ) + link_precs = [p for p in link_precs if p.name == "menuinst"] + [ + p for p in link_precs if p.name != "menuinst" + ] progressive_fetch_extract = ProgressiveFetchExtract(link_precs) progressive_fetch_extract.prepare() @@ -155,9 +156,7 @@ def install_actions(prefix, index, specs): channel_priority_map = prioritize_channels(LAST_CHANNEL_URLS) channels = IndexedSet(Channel(url) for url in channel_priority_map) subdirs = ( - IndexedSet( - subdir for subdir in (c.subdir for c in channels) if subdir - ) + IndexedSet(subdir for subdir in (c.subdir for c in channels) if subdir) or context.subdirs ) else: diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index 1f18d6e76f..77b97caa16 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -18,7 +18,6 @@ from conda.core.index import get_index from conda.core.prefix_data import PrefixData from conda.models.records import PrefixRecord -from conda.resolve import MatchSpec from . import conda_interface from .conda_interface import specs_from_args diff --git a/conda_build/post.py b/conda_build/post.py index af7289593b..433e0beedc 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -32,7 +32,7 @@ ) from pathlib import Path from subprocess import CalledProcessError, call, check_output -from typing import Iterable, Literal +from typing import Literal from conda.core.prefix_data import PrefixData from conda.models.records import PrefixRecord diff --git a/conda_build/render.py b/conda_build/render.py index c9a7ddef05..e7c0f652cb 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -41,7 +41,7 @@ from .utils import ( CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2, - dist_string_from_package_record + dist_string_from_package_record, ) from .variants import ( filter_by_key_value, @@ -354,7 +354,6 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # # this is to force the download # execute_actions(download_actions, index, verbose=m.config.debug) - pkg_files = {} packages = actions.get(LINK_ACTION, []) @@ -382,7 +381,9 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # TODO: this is a vile hack reaching into conda's internals. Replace with # proper conda API when available. if not pkg_loc: - pkg_record = [_ for _ in index if dist_string_from_package_record(_) == pkg_dist][0] + pkg_record = [ + rec for rec in index if dist_string_from_package_record(rec) == pkg_dist + ][0] pfe = ProgressiveFetchExtract(link_prefs=(pkg_record,)) with utils.LoggingContext(): pfe.execute() @@ -420,7 +421,10 @@ def get_upstream_pins(m: MetaData, actions, env): run_exports = pkg_data.get("run_exports", {}).get(pkg.version, {}) if run_exports is None: loc, dist = execute_download_actions( - m, actions, env=env, package_subset=[pkg], + m, + actions, + env=env, + package_subset=[pkg], )[pkg] run_exports = _read_specs_from_package(loc, dist) specs = _filter_run_exports(run_exports, ignore_list) diff --git a/conda_build/utils.py b/conda_build/utils.py index 3625ab5d2f..48495e18d0 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -53,8 +53,6 @@ CONDA_PACKAGE_EXTENSIONS, KNOWN_SUBDIRS, ) -from conda.core.prefix_data import PrefixData -from conda.models.records import PrefixRecord from .conda_interface import ( CondaHTTPError, From d1d195e52d07d4333d6ab587f8c46eaa2e857ec7 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 11:00:05 +0100 Subject: [PATCH 44/61] Use PackageRecord "prec" naming when unambiguous Signed-off-by: Marcel Bargull --- conda_build/environ.py | 6 +++--- conda_build/render.py | 49 +++++++++++++++++++++--------------------- conda_build/utils.py | 4 ++-- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 336f73c77b..114fe7db52 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -979,7 +979,7 @@ def get_install_actions( re.match(r"^%s(?:$|[\s=].*)" % pkg, str(dep)) for dep in specs ): actions[LINK_ACTION] = [ - spec for spec in actions[LINK_ACTION] if spec.name != pkg + prec for prec in actions[LINK_ACTION] if prec.name != pkg ] utils.trim_empty_keys(actions) cached_actions[(specs, env, subdir, channel_urls, disable_pip)] = actions.copy() @@ -1239,7 +1239,7 @@ def get_pinned_deps(m, section): channel_urls=tuple(m.config.channel_urls), ) runtime_deps = [ - " ".join(dist_string_from_package_record(link).rsplit("-", 2)) - for link in actions.get(LINK_ACTION, []) + " ".join(dist_string_from_package_record(prec).rsplit("-", 2)) + for prec in actions.get(LINK_ACTION, []) ] return runtime_deps diff --git a/conda_build/render.py b/conda_build/render.py index e7c0f652cb..ad797f24a3 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -92,8 +92,8 @@ def bldpkg_path(m): def actions_to_pins(actions): if LINK_ACTION in actions: return [ - " ".join(dist_string_from_package_record(spec).split()[0].rsplit("-", 2)) - for spec in actions[LINK_ACTION] + " ".join(dist_string_from_package_record(prec).split()[0].rsplit("-", 2)) + for prec in actions[LINK_ACTION] ] return [] @@ -356,23 +356,24 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files pkg_files = {} - packages = actions.get(LINK_ACTION, []) + precs = actions.get(LINK_ACTION, []) selected_packages = set() if package_subset: for pkg in package_subset: - if hasattr(pkg, "name"): - if pkg in packages: - selected_packages.add(pkg) + if isinstance(pkg, PackageRecord): + prec = pkg + if prec in precs: + selected_packages.add(prec) else: pkg_name = pkg.split()[0] - for link_pkg in packages: - if pkg_name == link_pkg.name: - selected_packages.add(link_pkg) + for link_prec in precs: + if pkg_name == link_prec.name: + selected_packages.add(link_prec) break - packages = selected_packages + precs = selected_packages - for pkg in packages: - pkg_dist = dist_string_from_package_record(pkg) + for prec in precs: + pkg_dist = dist_string_from_package_record(prec) pkg_loc = find_pkg_dir_or_file_in_pkgs_dirs( pkg_dist, m, files_only=require_files ) @@ -381,18 +382,18 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # TODO: this is a vile hack reaching into conda's internals. Replace with # proper conda API when available. if not pkg_loc: - pkg_record = [ + link_prec = [ rec for rec in index if dist_string_from_package_record(rec) == pkg_dist ][0] - pfe = ProgressiveFetchExtract(link_prefs=(pkg_record,)) + pfe = ProgressiveFetchExtract(link_prefs=(link_prec,)) with utils.LoggingContext(): pfe.execute() for pkg_dir in pkgs_dirs: - _loc = join(pkg_dir, pkg.fn) + _loc = join(pkg_dir, prec.fn) if isfile(_loc): pkg_loc = _loc break - pkg_files[pkg] = pkg_loc, pkg_dist + pkg_files[prec] = pkg_loc, pkg_dist return pkg_files @@ -403,29 +404,29 @@ def get_upstream_pins(m: MetaData, actions, env): env_specs = m.get_value(f"requirements/{env}", []) explicit_specs = [req.split(" ")[0] for req in env_specs] if env_specs else [] linked_packages = actions.get(LINK_ACTION, []) - linked_packages = [pkg for pkg in linked_packages if pkg.name in explicit_specs] + linked_packages = [prec for prec in linked_packages if prec.name in explicit_specs] ignore_pkgs_list = utils.ensure_list(m.get_value("build/ignore_run_exports_from")) ignore_list = utils.ensure_list(m.get_value("build/ignore_run_exports")) additional_specs = {} - for pkg in linked_packages: - if any(pkg.name in req.split(" ")[0] for req in ignore_pkgs_list): + for prec in linked_packages: + if any(prec.name in req.split(" ")[0] for req in ignore_pkgs_list): continue run_exports = None if m.config.use_channeldata: - channeldata = utils.download_channeldata(pkg.channel) + channeldata = utils.download_channeldata(prec.channel) # only use channeldata if requested, channeldata exists and contains # a packages key, otherwise use run_exports from the packages themselves if "packages" in channeldata: - pkg_data = channeldata["packages"].get(pkg.name, {}) - run_exports = pkg_data.get("run_exports", {}).get(pkg.version, {}) + pkg_data = channeldata["packages"].get(prec.name, {}) + run_exports = pkg_data.get("run_exports", {}).get(prec.version, {}) if run_exports is None: loc, dist = execute_download_actions( m, actions, env=env, - package_subset=[pkg], - )[pkg] + package_subset=[prec], + )[prec] run_exports = _read_specs_from_package(loc, dist) specs = _filter_run_exports(run_exports, ignore_list) if specs: diff --git a/conda_build/utils.py b/conda_build/utils.py index 48495e18d0..e913127ef3 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -2180,8 +2180,8 @@ def samefile(path1: Path, path2: Path) -> bool: return path1 == path2 -def dist_string_from_package_record(package_record): - string = package_record.fn +def dist_string_from_package_record(prec): + string = prec.fn if string.endswith("@"): raise NotImplementedError() From 558ac64f746a1a4c0f40c9af26722cc7a52136ba Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 11:09:47 +0100 Subject: [PATCH 45/61] Remove unnecessary prec->dist_name conversions Signed-off-by: Marcel Bargull --- conda_build/environ.py | 7 ++----- conda_build/render.py | 15 ++++++++------- conda_build/utils.py | 20 ++------------------ 3 files changed, 12 insertions(+), 30 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 114fe7db52..0947871abb 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -42,7 +42,7 @@ from .metadata import MetaData from .os_utils import external from .utils import ( - dist_string_from_package_record, + dist_dep_string, ensure_list, env_var, on_mac, @@ -1238,8 +1238,5 @@ def get_pinned_deps(m, section): output_folder=m.config.output_folder, channel_urls=tuple(m.config.channel_urls), ) - runtime_deps = [ - " ".join(dist_string_from_package_record(prec).rsplit("-", 2)) - for prec in actions.get(LINK_ACTION, []) - ] + runtime_deps = [dist_dep_string(prec) for prec in actions.get(LINK_ACTION, [])] return runtime_deps diff --git a/conda_build/render.py b/conda_build/render.py index ad797f24a3..da1f9a2b94 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -28,6 +28,7 @@ from . import environ, exceptions, source, utils from .conda_interface import ( + PackageRecord, ProgressiveFetchExtract, TemporaryDirectory, UnsatisfiableError, @@ -41,7 +42,7 @@ from .utils import ( CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2, - dist_string_from_package_record, + dist_dep_string, ) from .variants import ( filter_by_key_value, @@ -91,10 +92,7 @@ def bldpkg_path(m): def actions_to_pins(actions): if LINK_ACTION in actions: - return [ - " ".join(dist_string_from_package_record(prec).split()[0].rsplit("-", 2)) - for prec in actions[LINK_ACTION] - ] + return [dist_dep_string(prec) for prec in actions[LINK_ACTION]] return [] @@ -373,7 +371,7 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files precs = selected_packages for prec in precs: - pkg_dist = dist_string_from_package_record(prec) + pkg_dist = "-".join((prec.name, prec.version, prec.build)) pkg_loc = find_pkg_dir_or_file_in_pkgs_dirs( pkg_dist, m, files_only=require_files ) @@ -383,7 +381,10 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # proper conda API when available. if not pkg_loc: link_prec = [ - rec for rec in index if dist_string_from_package_record(rec) == pkg_dist + rec + for rec in index + if (rec.name, rec.version, rec.build) + == (prec.name, prec.version, prec.build) ][0] pfe = ProgressiveFetchExtract(link_prefs=(link_prec,)) with utils.LoggingContext(): diff --git a/conda_build/utils.py b/conda_build/utils.py index e913127ef3..f7704c94e9 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -2180,21 +2180,5 @@ def samefile(path1: Path, path2: Path) -> bool: return path1 == path2 -def dist_string_from_package_record(prec): - string = prec.fn - - if string.endswith("@"): - raise NotImplementedError() - - REGEX_STR = ( - r"(?:([^\s\[\]]+)::)?" # optional channel - r"([^\s\[\]]+)" # 3.x dist - r"(?:\[([a-zA-Z0-9_-]+)\])?" # with_features_depends - ) - channel, original_dist, w_f_d = re.search(REGEX_STR, string).groups() - - stripped = original_dist - for ext in CONDA_PACKAGE_EXTENSIONS: - if stripped.endswith(ext): - stripped = stripped[: -len(ext)] - return stripped +def dist_dep_string(prec: PackageRecord) -> str: + return " ".join((prec.name, prec.version, prec.build)).rstrip() From 27487741b34d6536e149ea7e184dc7eb168f2939 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 14:28:36 +0100 Subject: [PATCH 46/61] Use callback=reset_context in install_actions Port of fix from https://github.com/conda/conda/pull/13357 . Signed-off-by: Marcel Bargull --- conda_build/environ_actions.py | 4 ++-- tests/test_api_build.py | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/conda_build/environ_actions.py b/conda_build/environ_actions.py index fbfaacff41..57793e0bc5 100644 --- a/conda_build/environ_actions.py +++ b/conda_build/environ_actions.py @@ -22,7 +22,7 @@ DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL, ) -from conda.base.context import context, stack_context_default +from conda.base.context import context, reset_context from conda.common.io import env_vars from conda.core.index import LAST_CHANNEL_URLS from conda.core.link import PrefixSetup, UnlinkLinkTransaction @@ -149,7 +149,7 @@ def install_actions(prefix, index, specs): "CONDA_ALLOW_NON_CHANNEL_URLS": "true", "CONDA_SOLVER_IGNORE_TIMESTAMPS": "false", }, - stack_callback=stack_context_default, + callback=reset_context, ): # a hack since in conda-build we don't track channel_priority_map if LAST_CHANNEL_URLS: diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 1ac2ca06d2..639fefc3e7 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -32,10 +32,8 @@ from conda_build import __version__, api, exceptions from conda_build.conda_interface import ( - CONDA_VERSION, CondaError, LinkError, - VersionOrder, context, reset_context, url_path, @@ -1948,16 +1946,6 @@ def test_add_pip_as_python_dependency_from_condarc_file( Test whether settings from .condarc files are heeded. ref: https://github.com/conda/conda-libmamba-solver/issues/393 """ - if VersionOrder(CONDA_VERSION) <= VersionOrder("23.10.0"): - if not add_pip_as_python_dependency and context.solver == "libmamba": - pytest.xfail( - "conda.plan.install_actions from conda<=23.10.0 ignores .condarc files." - ) - from conda.base.context import context_stack - - # ContextStack's pop/replace methods don't call self.apply. - context_stack.apply() - # TODO: SubdirData._cache_ clearing might not be needed for future conda versions. # See https://github.com/conda/conda/pull/13365 for proposed changes. from conda.core.subdir_data import SubdirData From 6065333e4d1553c1972d728d830ed57cbfcae078 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 19 Jan 2024 17:28:56 +0100 Subject: [PATCH 47/61] Move install actions to .environ Signed-off-by: Marcel Bargull --- conda_build/environ.py | 174 +++++++++++++++++++++++++++++-- conda_build/environ_actions.py | 182 --------------------------------- conda_build/render.py | 2 +- 3 files changed, 168 insertions(+), 190 deletions(-) delete mode 100644 conda_build/environ_actions.py diff --git a/conda_build/environ.py b/conda_build/environ.py index 0947871abb..c67c7d950f 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -10,32 +10,47 @@ import subprocess import sys import warnings +from collections import defaultdict from functools import lru_cache from glob import glob +from logging import getLogger from os.path import join, normpath +try: + from boltons.setutils import IndexedSet +except ImportError: # pragma: no cover + from conda._vendor.boltons.setutils import IndexedSet + +from conda.base.constants import ( + DEFAULTS_CHANNEL_NAME, + UNKNOWN_CHANNEL, +) +from conda.common.io import env_vars +from conda.core.index import LAST_CHANNEL_URLS +from conda.core.link import PrefixSetup, UnlinkLinkTransaction +from conda.core.prefix_data import PrefixData +from conda.models.channel import prioritize_channels + from . import utils from .conda_interface import ( + Channel, CondaError, LinkError, LockError, + MatchSpec, NoPackagesFoundError, + PackageRecord, PaddingError, + ProgressiveFetchExtract, TemporaryDirectory, UnsatisfiableError, + context, create_default_packages, get_version_from_git_tag, pkgs_dirs, reset_context, root_dir, ) -from .environ_actions import ( - LINK_ACTION, - PREFIX_ACTION, - display_actions, - execute_actions, - install_actions, -) from .exceptions import BuildLockError, DependencyNeedsBuildingError from .features import feature_list from .index import get_build_index @@ -51,6 +66,11 @@ ) from .variants import get_default_variant +log = getLogger(__name__) + +PREFIX_ACTION = "PREFIX" +LINK_ACTION = "LINK" + # these are things that we provide env vars for more explicitly. This list disables the # pass-through of variant values to env vars for these keys. LANGUAGES = ("PERL", "LUA", "R", "NUMPY", "PYTHON") @@ -1240,3 +1260,143 @@ def get_pinned_deps(m, section): ) runtime_deps = [dist_dep_string(prec) for prec in actions.get(LINK_ACTION, [])] return runtime_deps + + +def install_actions(prefix, index, specs): + with env_vars( + { + "CONDA_ALLOW_NON_CHANNEL_URLS": "true", + "CONDA_SOLVER_IGNORE_TIMESTAMPS": "false", + }, + callback=reset_context, + ): + # a hack since in conda-build we don't track channel_priority_map + if LAST_CHANNEL_URLS: + channel_priority_map = prioritize_channels(LAST_CHANNEL_URLS) + channels = IndexedSet(Channel(url) for url in channel_priority_map) + subdirs = ( + IndexedSet(subdir for subdir in (c.subdir for c in channels) if subdir) + or context.subdirs + ) + else: + channels = subdirs = None + + specs = tuple(MatchSpec(spec) for spec in specs) + + PrefixData._cache_.clear() + + solver_backend = context.plugin_manager.get_cached_solver_backend() + solver = solver_backend(prefix, channels, subdirs, specs_to_add=specs) + if index: + # Solver can modify the index (e.g., Solver._prepare adds virtual + # package) => Copy index (just outer container, not deep copy) + # to conserve it. + solver._index = index.copy() + txn = solver.solve_for_transaction(prune=False, ignore_pinned=False) + prefix_setup = txn.prefix_setups[prefix] + actions = { + PREFIX_ACTION: prefix, + LINK_ACTION: [prec for prec in prefix_setup.link_precs], + } + return actions + + +def execute_actions(actions): + assert PREFIX_ACTION in actions and actions[PREFIX_ACTION] + prefix = actions[PREFIX_ACTION] + + if LINK_ACTION not in actions: + log.debug(f"action {LINK_ACTION} not in actions") + return + + link_precs = actions[LINK_ACTION] + if not link_precs: + log.debug(f"action {LINK_ACTION} has None value") + return + + if on_win: + # Always link menuinst first/last on windows in case a subsequent + # package tries to import it to create/remove a shortcut + link_precs = [p for p in link_precs if p.name == "menuinst"] + [ + p for p in link_precs if p.name != "menuinst" + ] + + progressive_fetch_extract = ProgressiveFetchExtract(link_precs) + progressive_fetch_extract.prepare() + + stp = PrefixSetup(prefix, (), link_precs, (), [], ()) + unlink_link_transaction = UnlinkLinkTransaction(stp) + + log.debug(" %s(%r)", "PROGRESSIVEFETCHEXTRACT", progressive_fetch_extract) + progressive_fetch_extract.execute() + log.debug(" %s(%r)", "UNLINKLINKTRANSACTION", unlink_link_transaction) + unlink_link_transaction.execute() + + +def display_actions(actions): + prefix = actions.get(PREFIX_ACTION) + builder = ["", "## Package Plan ##\n"] + if prefix: + builder.append(" environment location: %s" % prefix) + builder.append("") + print("\n".join(builder)) + + show_channel_urls = context.show_channel_urls + + def channel_str(rec): + if rec.get("schannel"): + return rec["schannel"] + if rec.get("url"): + return Channel(rec["url"]).canonical_name + if rec.get("channel"): + return Channel(rec["channel"]).canonical_name + return UNKNOWN_CHANNEL + + def channel_filt(s): + if show_channel_urls is False: + return "" + if show_channel_urls is None and s == DEFAULTS_CHANNEL_NAME: + return "" + return s + + packages = defaultdict(lambda: "") + features = defaultdict(lambda: "") + channels = defaultdict(lambda: "") + + for prec in actions.get(LINK_ACTION, []): + assert isinstance(prec, PackageRecord) + pkg = prec["name"] + channels[pkg] = channel_filt(channel_str(prec)) + packages[pkg] = prec["version"] + "-" + prec["build"] + features[pkg] = ",".join(prec.get("features") or ()) + + fmt = {} + if packages: + maxpkg = max(len(p) for p in packages) + 1 + maxver = max(len(p) for p in packages.values()) + maxfeatures = max(len(p) for p in features.values()) + maxchannels = max(len(p) for p in channels.values()) + for pkg in packages: + # That's right. I'm using old-style string formatting to generate a + # string with new-style string formatting. + fmt[pkg] = f"{{pkg:<{maxpkg}}} {{vers:<{maxver}}}" + if maxchannels: + fmt[pkg] += " {channel:<%s}" % maxchannels + if features[pkg]: + fmt[pkg] += " [{features:<%s}]" % maxfeatures + + lead = " " * 4 + + def format(s, pkg): + return lead + s.format( + pkg=pkg + ":", + vers=packages[pkg], + channel=channels[pkg], + features=features[pkg], + ) + + if packages: + print("\nThe following NEW packages will be INSTALLED:\n") + for pkg in sorted(packages): + print(format(fmt[pkg], pkg)) + print() diff --git a/conda_build/environ_actions.py b/conda_build/environ_actions.py deleted file mode 100644 index 57793e0bc5..0000000000 --- a/conda_build/environ_actions.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright (C) 2014 Anaconda, Inc -# SPDX-License-Identifier: BSD-3-Clause -""" -Handle the planning of installs and their execution. - -NOTE: - conda.install uses canonical package names in its interface functions, - whereas conda.resolve uses package filenames, as those are used as index - keys. We try to keep fixes to this "impedance mismatch" local to this - module. -""" - -from collections import defaultdict -from logging import getLogger - -try: - from boltons.setutils import IndexedSet -except ImportError: # pragma: no cover - from conda._vendor.boltons.setutils import IndexedSet - -from conda.base.constants import ( - DEFAULTS_CHANNEL_NAME, - UNKNOWN_CHANNEL, -) -from conda.base.context import context, reset_context -from conda.common.io import env_vars -from conda.core.index import LAST_CHANNEL_URLS -from conda.core.link import PrefixSetup, UnlinkLinkTransaction -from conda.core.prefix_data import PrefixData -from conda.exports import ( - Channel, - MatchSpec, - PackageRecord, - ProgressiveFetchExtract, - on_win, -) -from conda.models.channel import prioritize_channels - -PREFIX_ACTION = "PREFIX" -LINK_ACTION = "LINK" - -log = getLogger(__name__) - - -def display_actions(actions): - prefix = actions.get(PREFIX_ACTION) - builder = ["", "## Package Plan ##\n"] - if prefix: - builder.append(" environment location: %s" % prefix) - builder.append("") - print("\n".join(builder)) - - show_channel_urls = context.show_channel_urls - - def channel_str(rec): - if rec.get("schannel"): - return rec["schannel"] - if rec.get("url"): - return Channel(rec["url"]).canonical_name - if rec.get("channel"): - return Channel(rec["channel"]).canonical_name - return UNKNOWN_CHANNEL - - def channel_filt(s): - if show_channel_urls is False: - return "" - if show_channel_urls is None and s == DEFAULTS_CHANNEL_NAME: - return "" - return s - - packages = defaultdict(lambda: "") - features = defaultdict(lambda: "") - channels = defaultdict(lambda: "") - - for prec in actions.get(LINK_ACTION, []): - assert isinstance(prec, PackageRecord) - pkg = prec["name"] - channels[pkg] = channel_filt(channel_str(prec)) - packages[pkg] = prec["version"] + "-" + prec["build"] - features[pkg] = ",".join(prec.get("features") or ()) - - fmt = {} - if packages: - maxpkg = max(len(p) for p in packages) + 1 - maxver = max(len(p) for p in packages.values()) - maxfeatures = max(len(p) for p in features.values()) - maxchannels = max(len(p) for p in channels.values()) - for pkg in packages: - # That's right. I'm using old-style string formatting to generate a - # string with new-style string formatting. - fmt[pkg] = f"{{pkg:<{maxpkg}}} {{vers:<{maxver}}}" - if maxchannels: - fmt[pkg] += " {channel:<%s}" % maxchannels - if features[pkg]: - fmt[pkg] += " [{features:<%s}]" % maxfeatures - - lead = " " * 4 - - def format(s, pkg): - return lead + s.format( - pkg=pkg + ":", - vers=packages[pkg], - channel=channels[pkg], - features=features[pkg], - ) - - if packages: - print("\nThe following NEW packages will be INSTALLED:\n") - for pkg in sorted(packages): - print(format(fmt[pkg], pkg)) - print() - - -def execute_actions(actions): - assert PREFIX_ACTION in actions and actions[PREFIX_ACTION] - prefix = actions[PREFIX_ACTION] - - if LINK_ACTION not in actions: - log.debug(f"action {LINK_ACTION} not in actions") - return - - link_precs = actions[LINK_ACTION] - if not link_precs: - log.debug(f"action {LINK_ACTION} has None value") - return - - if on_win: - # Always link menuinst first/last on windows in case a subsequent - # package tries to import it to create/remove a shortcut - link_precs = [p for p in link_precs if p.name == "menuinst"] + [ - p for p in link_precs if p.name != "menuinst" - ] - - progressive_fetch_extract = ProgressiveFetchExtract(link_precs) - progressive_fetch_extract.prepare() - - stp = PrefixSetup(prefix, (), link_precs, (), [], ()) - unlink_link_transaction = UnlinkLinkTransaction(stp) - - log.debug(" %s(%r)", "PROGRESSIVEFETCHEXTRACT", progressive_fetch_extract) - progressive_fetch_extract.execute() - log.debug(" %s(%r)", "UNLINKLINKTRANSACTION", unlink_link_transaction) - unlink_link_transaction.execute() - - -def install_actions(prefix, index, specs): - with env_vars( - { - "CONDA_ALLOW_NON_CHANNEL_URLS": "true", - "CONDA_SOLVER_IGNORE_TIMESTAMPS": "false", - }, - callback=reset_context, - ): - # a hack since in conda-build we don't track channel_priority_map - if LAST_CHANNEL_URLS: - channel_priority_map = prioritize_channels(LAST_CHANNEL_URLS) - channels = IndexedSet(Channel(url) for url in channel_priority_map) - subdirs = ( - IndexedSet(subdir for subdir in (c.subdir for c in channels) if subdir) - or context.subdirs - ) - else: - channels = subdirs = None - - specs = tuple(MatchSpec(spec) for spec in specs) - - PrefixData._cache_.clear() - - solver_backend = context.plugin_manager.get_cached_solver_backend() - solver = solver_backend(prefix, channels, subdirs, specs_to_add=specs) - if index: - # Solver can modify the index (e.g., Solver._prepare adds virtual - # package) => Copy index (just outer container, not deep copy) - # to conserve it. - solver._index = index.copy() - txn = solver.solve_for_transaction(prune=False, ignore_pinned=False) - prefix_setup = txn.prefix_setups[prefix] - actions = { - PREFIX_ACTION: prefix, - LINK_ACTION: [prec for prec in prefix_setup.link_precs], - } - return actions diff --git a/conda_build/render.py b/conda_build/render.py index da1f9a2b94..b6ddaeeeaa 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -35,7 +35,7 @@ pkgs_dirs, specs_from_url, ) -from .environ_actions import LINK_ACTION +from .environ import LINK_ACTION from .exceptions import DependencyNeedsBuildingError from .index import get_build_index from .metadata import MetaData, combine_top_level_metadata_with_output From 0c658fb1355bbe22d4486d2c9dcaa0ffb5fdf7e4 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Thu, 25 Jan 2024 10:42:45 +0100 Subject: [PATCH 48/61] Remove now unused lru_cache import Signed-off-by: Marcel Bargull --- conda_build/inspect_pkg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index 7cb7653b69..c8da76cb16 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -6,7 +6,6 @@ import os import sys from collections import defaultdict -from functools import lru_cache from itertools import groupby from operator import itemgetter from os.path import abspath, basename, dirname, exists, join, normcase From 2cb1f65f545de9791559332d23b308c71acc0b62 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Thu, 25 Jan 2024 21:32:52 +0100 Subject: [PATCH 49/61] Add news entry for Dist removal Signed-off-by: Marcel Bargull --- news/5074-dist-removal | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 news/5074-dist-removal diff --git a/news/5074-dist-removal b/news/5074-dist-removal new file mode 100644 index 0000000000..3022c458a4 --- /dev/null +++ b/news/5074-dist-removal @@ -0,0 +1,27 @@ +### Enhancements + +* + +### Bug fixes + +* + +### Deprecations + +* Removed `conda_build.conda_interface.get_installed_version`. (#5074) +* Removed `conda_build.environ.clean_pkg_cache`. (#5074) +* Removed `conda_build.inspect_pkg.dist_files`. (#5074) +* Removed `conda_build.inspect_pkg._installed`. (#5074) +* Removed `conda_build.post.dists_from_names`. (#5074) +* Removed `conda_build.post.FakeDist`. (#5074) +* Removed `conda_build.post._get_fake_pkg_dist`. (#5074) + +### Docs + +* + +### Other + +* Removed dependency on `conda.plan`. (#5074) +* Removed almost all dependency on `conda.models.dist`. (#5074) +* Replaced usage of legacy `conda.models.dist.Dist` by `conda.models.records.PackageRecord`. (#5074) From 4b726cb1f0dfba56ca7e18241c0b75448f8fb3d4 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Thu, 25 Jan 2024 21:54:42 +0100 Subject: [PATCH 50/61] Add back re-exported symbols and mark deprecated Signed-off-by: Marcel Bargull --- conda_build/conda_interface.py | 18 +++++++++++++++++- conda_build/index.py | 2 +- conda_build/utils.py | 18 ++++++++++++++++++ news/5074-dist-removal | 8 ++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 6e2412947b..0c82586632 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -15,7 +15,6 @@ ) from conda.base.context import context, determine_target_prefix, reset_context from conda.base.context import non_x86_machines as non_x86_linux_machines # noqa: F401 -from conda.core.index import get_index # noqa: F401 from conda.core.package_cache import ProgressiveFetchExtract # noqa: F401 from conda.core.package_cache_data import PackageCacheData # noqa: F401 from conda.exceptions import ( # noqa: F401 @@ -47,15 +46,23 @@ _toposort, add_parser_channels, add_parser_prefix, + display_actions, download, + execute_actions, + execute_plan, + get_index, handle_proxy_407, hashsum_file, human_bytes, input, + install_actions, lchmod, + linked, + linked_data, md5_file, memoized, normalized_version, + package_cache, prefix_placeholder, rm_rf, spec_from_line, @@ -69,6 +76,7 @@ win_path_to_unix, ) from conda.models.channel import get_conda_build_local_url # noqa: F401 +from conda.models.dist import Dist from conda.models.records import PackageRecord, PrefixRecord from .deprecations import deprecated @@ -80,6 +88,14 @@ PackageRecord, addendum="Use `conda.models.records.PackageRecord` instead.", ) +deprecated.constant("24.1.0", "24.3.0", "Dist", Dist) +deprecated.constant("24.1.0", "24.3.0", "display_actions", display_actions) +deprecated.constant("24.1.0", "24.3.0", "execute_actions", execute_actions) +deprecated.constant("24.1.0", "24.3.0", "execute_plan", execute_plan) +deprecated.constant("24.1.0", "24.3.0", "get_index", get_index) +deprecated.constant("24.1.0", "24.3.0", "install_actions", install_actions) +deprecated.constant("24.1.0", "24.3.0", "linked", linked) +deprecated.constant("24.1.0", "24.3.0", "linked_data", linked_data) # TODO: Go to references of all properties below and import them from `context` instead binstar_upload = context.binstar_upload diff --git a/conda_build/index.py b/conda_build/index.py index 37b0fe0cd9..5b513e2dc0 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -37,6 +37,7 @@ from conda.common.compat import ensure_binary # BAD BAD BAD - conda internals +from conda.core.index import get_index from conda.core.subdir_data import SubdirData from conda.models.channel import Channel from conda_index.index import update_index as _update_index @@ -57,7 +58,6 @@ TemporaryDirectory, VersionOrder, context, - get_index, human_bytes, url_path, ) diff --git a/conda_build/utils.py b/conda_build/utils.py index f7704c94e9..6c5a9a1de0 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -53,6 +53,9 @@ CONDA_PACKAGE_EXTENSIONS, KNOWN_SUBDIRS, ) +from conda.core.prefix_data import PrefixData +from conda.models.dist import Dist +from conda.models.records import PrefixRecord from .conda_interface import ( CondaHTTPError, @@ -2127,6 +2130,21 @@ def download_channeldata(channel_url): return data +@deprecated("24.1.0", "24.3.0") +def linked_data_no_multichannels( + prefix: str | os.PathLike | Path, +) -> dict[Dist, PrefixRecord]: + """ + Return a dictionary of the linked packages in prefix, with correct channels, hopefully. + cc @kalefranz. + """ + prefix = Path(prefix) + return { + Dist.from_string(prec.fn, channel_override=prec.channel.name): prec + for prec in PrefixData(str(prefix)).iter_records() + } + + def shutil_move_more_retrying(src, dest, debug_name): log = get_logger(__name__) log.info(f"Renaming {debug_name} directory '{src}' to '{dest}'") diff --git a/news/5074-dist-removal b/news/5074-dist-removal index 3022c458a4..3d759fd06d 100644 --- a/news/5074-dist-removal +++ b/news/5074-dist-removal @@ -15,6 +15,14 @@ * Removed `conda_build.post.dists_from_names`. (#5074) * Removed `conda_build.post.FakeDist`. (#5074) * Removed `conda_build.post._get_fake_pkg_dist`. (#5074) +* Mark `conda_build.conda_interface.Dist` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.display_actions` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.execute_actions` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.execute_plan` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.get_index` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.install_actions` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.linked` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.linked_data` as pending deprecation. (#5074) ### Docs From 0b4dd86e90cc29d1bc5ba83d19d2efa805fac4da Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Thu, 25 Jan 2024 22:01:06 +0100 Subject: [PATCH 51/61] Remove unused PackageCacheData import Signed-off-by: Marcel Bargull --- conda_build/conda_interface.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 0c82586632..9b22f67552 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -16,7 +16,6 @@ from conda.base.context import context, determine_target_prefix, reset_context from conda.base.context import non_x86_machines as non_x86_linux_machines # noqa: F401 from conda.core.package_cache import ProgressiveFetchExtract # noqa: F401 -from conda.core.package_cache_data import PackageCacheData # noqa: F401 from conda.exceptions import ( # noqa: F401 CondaError, CondaHTTPError, From 94974edf133e5f07df3c5c353d3b9ca570a5cdf9 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Thu, 25 Jan 2024 22:09:33 +0100 Subject: [PATCH 52/61] Use less cryptic function name Signed-off-by: Marcel Bargull --- conda_build/environ.py | 6 ++++-- conda_build/render.py | 4 ++-- conda_build/utils.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index c67c7d950f..b8ebefa66e 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -57,11 +57,11 @@ from .metadata import MetaData from .os_utils import external from .utils import ( - dist_dep_string, ensure_list, env_var, on_mac, on_win, + package_record_to_requirement, prepend_bin_path, ) from .variants import get_default_variant @@ -1258,7 +1258,9 @@ def get_pinned_deps(m, section): output_folder=m.config.output_folder, channel_urls=tuple(m.config.channel_urls), ) - runtime_deps = [dist_dep_string(prec) for prec in actions.get(LINK_ACTION, [])] + runtime_deps = [ + package_record_to_requirement(prec) for prec in actions.get(LINK_ACTION, []) + ] return runtime_deps diff --git a/conda_build/render.py b/conda_build/render.py index b6ddaeeeaa..fdaac1bef5 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -42,7 +42,7 @@ from .utils import ( CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2, - dist_dep_string, + package_record_to_requirement, ) from .variants import ( filter_by_key_value, @@ -92,7 +92,7 @@ def bldpkg_path(m): def actions_to_pins(actions): if LINK_ACTION in actions: - return [dist_dep_string(prec) for prec in actions[LINK_ACTION]] + return [package_record_to_requirement(prec) for prec in actions[LINK_ACTION]] return [] diff --git a/conda_build/utils.py b/conda_build/utils.py index 6c5a9a1de0..03a322caea 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -2198,5 +2198,5 @@ def samefile(path1: Path, path2: Path) -> bool: return path1 == path2 -def dist_dep_string(prec: PackageRecord) -> str: +def package_record_to_requirement(prec: PackageRecord) -> str: return " ".join((prec.name, prec.version, prec.build)).rstrip() From c13b476d66523f39214e91c6300f7b063157576f Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Thu, 25 Jan 2024 22:15:22 +0100 Subject: [PATCH 53/61] Add back handling of single-record package_subset in render.execute_download_actions Signed-off-by: Marcel Bargull --- conda_build/render.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda_build/render.py b/conda_build/render.py index fdaac1bef5..ea1c9074ab 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -355,6 +355,10 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files pkg_files = {} precs = actions.get(LINK_ACTION, []) + if isinstance(package_subset, PackageRecord): + package_subset = [package_subset] + else: + package_subset = utils.ensure_list(package_subset) selected_packages = set() if package_subset: for pkg in package_subset: From 0e0c45069bd5fe4bbfe38f69ce875a1728f75b1a Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Thu, 25 Jan 2024 22:22:12 +0100 Subject: [PATCH 54/61] Mark functions from conda.plan as non-public These will be changed/superseded/removed in a future version and thus should not be used elsewhere. Signed-off-by: Marcel Bargull --- conda_build/environ.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index b8ebefa66e..aae10dff51 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -924,7 +924,7 @@ def get_install_actions( with utils.LoggingContext(conda_log_level): with capture(): try: - actions = install_actions(prefix, index, specs) + actions = _install_actions(prefix, index, specs) except (NoPackagesFoundError, UnsatisfiableError) as exc: raise DependencyNeedsBuildingError(exc, subdir=subdir) except ( @@ -1076,13 +1076,13 @@ def create_env( timeout=config.timeout, ) utils.trim_empty_keys(actions) - display_actions(actions) + _display_actions(actions) if utils.on_win: for k, v in os.environ.items(): os.environ[k] = str(v) with env_var("CONDA_QUIET", not config.verbose, reset_context): with env_var("CONDA_JSON", not config.verbose, reset_context): - execute_actions(actions) + _execute_actions(actions) except ( SystemExit, PaddingError, @@ -1264,6 +1264,9 @@ def get_pinned_deps(m, section): return runtime_deps +# NOTE: The function has to retain the "install_actions" name for now since +# conda_libmamba_solver.solver.LibMambaSolver._called_from_conda_build +# checks for this name in the call stack explicitly. def install_actions(prefix, index, specs): with env_vars( { @@ -1303,7 +1306,11 @@ def install_actions(prefix, index, specs): return actions -def execute_actions(actions): +_install_actions = install_actions +del install_actions + + +def _execute_actions(actions): assert PREFIX_ACTION in actions and actions[PREFIX_ACTION] prefix = actions[PREFIX_ACTION] @@ -1335,7 +1342,7 @@ def execute_actions(actions): unlink_link_transaction.execute() -def display_actions(actions): +def _display_actions(actions): prefix = actions.get(PREFIX_ACTION) builder = ["", "## Package Plan ##\n"] if prefix: From f0652fe569b4ff5b76dd4e95ea80c07282588668 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 26 Jan 2024 06:35:07 +0100 Subject: [PATCH 55/61] Remove dependency on boltons.setutils.IndexedSet Signed-off-by: Marcel Bargull --- conda_build/environ.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index aae10dff51..3cc61eb4fa 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -16,11 +16,6 @@ from logging import getLogger from os.path import join, normpath -try: - from boltons.setutils import IndexedSet -except ImportError: # pragma: no cover - from conda._vendor.boltons.setutils import IndexedSet - from conda.base.constants import ( DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL, @@ -1278,9 +1273,16 @@ def install_actions(prefix, index, specs): # a hack since in conda-build we don't track channel_priority_map if LAST_CHANNEL_URLS: channel_priority_map = prioritize_channels(LAST_CHANNEL_URLS) - channels = IndexedSet(Channel(url) for url in channel_priority_map) + # tuple(dict.fromkeys(...)) removes duplicates while preserving input order. + channels = tuple( + dict.fromkeys(Channel(url) for url in channel_priority_map) + ) subdirs = ( - IndexedSet(subdir for subdir in (c.subdir for c in channels) if subdir) + tuple( + dict.fromkeys( + subdir for subdir in (c.subdir for c in channels) if subdir + ) + ) or context.subdirs ) else: From ce836aa29a72a76636b3daffa485c6c25a2a53f9 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 26 Jan 2024 06:46:54 +0100 Subject: [PATCH 56/61] Add notes to environ.*_actions' conda.plan source Also apply menuinst handling change from conda=23.11 since the code was originally copied from conda=23.10. Signed-off-by: Marcel Bargull --- conda_build/environ.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 3cc61eb4fa..17ec237f30 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -1263,6 +1263,9 @@ def get_pinned_deps(m, section): # conda_libmamba_solver.solver.LibMambaSolver._called_from_conda_build # checks for this name in the call stack explicitly. def install_actions(prefix, index, specs): + # This is copied over from https://github.com/conda/conda/blob/23.11.0/conda/plan.py#L471 + # but reduced to only the functionality actually used within conda-build. + with env_vars( { "CONDA_ALLOW_NON_CHANNEL_URLS": "true", @@ -1313,6 +1316,9 @@ def install_actions(prefix, index, specs): def _execute_actions(actions): + # This is copied over from https://github.com/conda/conda/blob/23.11.0/conda/plan.py#L575 + # but reduced to only the functionality actually used within conda-build. + assert PREFIX_ACTION in actions and actions[PREFIX_ACTION] prefix = actions[PREFIX_ACTION] @@ -1325,12 +1331,11 @@ def _execute_actions(actions): log.debug(f"action {LINK_ACTION} has None value") return - if on_win: - # Always link menuinst first/last on windows in case a subsequent - # package tries to import it to create/remove a shortcut - link_precs = [p for p in link_precs if p.name == "menuinst"] + [ - p for p in link_precs if p.name != "menuinst" - ] + # Always link menuinst first/last on windows in case a subsequent + # package tries to import it to create/remove a shortcut + link_precs = [p for p in link_precs if p.name == "menuinst"] + [ + p for p in link_precs if p.name != "menuinst" + ] progressive_fetch_extract = ProgressiveFetchExtract(link_precs) progressive_fetch_extract.prepare() @@ -1345,6 +1350,9 @@ def _execute_actions(actions): def _display_actions(actions): + # This is copied over from https://github.com/conda/conda/blob/23.11.0/conda/plan.py#L58 + # but reduced to only the functionality actually used within conda-build. + prefix = actions.get(PREFIX_ACTION) builder = ["", "## Package Plan ##\n"] if prefix: From 155c15669731f4475dd6ee8be3c9bab19503c411 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 26 Jan 2024 06:55:08 +0100 Subject: [PATCH 57/61] Fix deprecated.constant usage in conda_interface Signed-off-by: Marcel Bargull --- conda_build/conda_interface.py | 35 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 9b22f67552..40e761af28 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -45,23 +45,15 @@ _toposort, add_parser_channels, add_parser_prefix, - display_actions, download, - execute_actions, - execute_plan, - get_index, handle_proxy_407, hashsum_file, human_bytes, input, - install_actions, lchmod, - linked, - linked_data, md5_file, memoized, normalized_version, - package_cache, prefix_placeholder, rm_rf, spec_from_line, @@ -74,8 +66,16 @@ walk_prefix, win_path_to_unix, ) +from conda.exports import display_actions as _display_actions +from conda.exports import execute_actions as _execute_actions +from conda.exports import execute_plan as _execute_plan +from conda.exports import get_index as _get_index +from conda.exports import install_actions as _install_actions +from conda.exports import linked as _linked +from conda.exports import linked_data as _linked_data +from conda.exports import package_cache as _package_cache from conda.models.channel import get_conda_build_local_url # noqa: F401 -from conda.models.dist import Dist +from conda.models.dist import Dist as _Dist from conda.models.records import PackageRecord, PrefixRecord from .deprecations import deprecated @@ -87,14 +87,15 @@ PackageRecord, addendum="Use `conda.models.records.PackageRecord` instead.", ) -deprecated.constant("24.1.0", "24.3.0", "Dist", Dist) -deprecated.constant("24.1.0", "24.3.0", "display_actions", display_actions) -deprecated.constant("24.1.0", "24.3.0", "execute_actions", execute_actions) -deprecated.constant("24.1.0", "24.3.0", "execute_plan", execute_plan) -deprecated.constant("24.1.0", "24.3.0", "get_index", get_index) -deprecated.constant("24.1.0", "24.3.0", "install_actions", install_actions) -deprecated.constant("24.1.0", "24.3.0", "linked", linked) -deprecated.constant("24.1.0", "24.3.0", "linked_data", linked_data) +deprecated.constant("24.1.0", "24.3.0", "Dist", _Dist) +deprecated.constant("24.1.0", "24.3.0", "display_actions", _display_actions) +deprecated.constant("24.1.0", "24.3.0", "execute_actions", _execute_actions) +deprecated.constant("24.1.0", "24.3.0", "execute_plan", _execute_plan) +deprecated.constant("24.1.0", "24.3.0", "get_index", _get_index) +deprecated.constant("24.1.0", "24.3.0", "install_actions", _install_actions) +deprecated.constant("24.1.0", "24.3.0", "linked", _linked) +deprecated.constant("24.1.0", "24.3.0", "linked_data", _linked_data) +deprecated.constant("24.1.0", "24.3.0", "package_cache", _package_cache) # TODO: Go to references of all properties below and import them from `context` instead binstar_upload = context.binstar_upload From e98e96dcf380c9ecd944e4e1adf6ac2ef79e34f0 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 26 Jan 2024 07:18:58 +0100 Subject: [PATCH 58/61] Apply style suggestions from code review Co-authored-by: Ken Odegard --- conda_build/render.py | 2 +- conda_build/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/render.py b/conda_build/render.py index ea1c9074ab..c75838a65b 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -342,7 +342,7 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files # this should be just downloading packages. We don't need to extract them - - # NOTE: The following out-commented execute_actions was defunct + # NOTE: The following commented execute_actions is defunct # (FETCH/EXTRACT were replaced by PROGRESSIVEFETCHEXTRACT). # # download_actions = { diff --git a/conda_build/utils.py b/conda_build/utils.py index 03a322caea..b1cd33608f 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -2199,4 +2199,4 @@ def samefile(path1: Path, path2: Path) -> bool: def package_record_to_requirement(prec: PackageRecord) -> str: - return " ".join((prec.name, prec.version, prec.build)).rstrip() + return f"{prec.name} {prec.version} {prec.build}".rstrip() From 6a5359a8ea3d9f1e26a8f194ca05d659baa93548 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 26 Jan 2024 08:22:42 +0100 Subject: [PATCH 59/61] Remove rstrip from package_record_to_requirements The replaced code did not rstrip, so let us keep it that way for now. Signed-off-by: Marcel Bargull --- conda_build/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index b1cd33608f..da21c54b51 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -2199,4 +2199,4 @@ def samefile(path1: Path, path2: Path) -> bool: def package_record_to_requirement(prec: PackageRecord) -> str: - return f"{prec.name} {prec.version} {prec.build}".rstrip() + return f"{prec.name} {prec.version} {prec.build}" From c6c04549b10a8750289e2580e0fecac658cdd469 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 26 Jan 2024 17:28:33 +0100 Subject: [PATCH 60/61] Merge news/5074-dist-removal into CHANGELOG.md Signed-off-by: Marcel Bargull --- CHANGELOG.md | 12 ++++++++++++ news/5074-dist-removal | 35 ----------------------------------- 2 files changed, 12 insertions(+), 35 deletions(-) delete mode 100644 news/5074-dist-removal diff --git a/CHANGELOG.md b/CHANGELOG.md index a1937ae4c4..aac41a7ddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,15 @@ * Mark `conda_build.inspect_pkg.check_install(package)` as pending deprecation in favor of `conda_build.inspect_pkg.check_install(subdir)`. (#5033) * Mark `conda_build.inspect_pkg.check_install(prepend)` as pending deprecation. (#5033) * Mark `conda_build.inspect_pkg.check_install(minimal_hint)` as pending deprecation. (#5033) +* Mark `conda_build.conda_interface.Dist` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.display_actions` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.execute_actions` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.execute_plan` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.get_index` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.install_actions` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.linked` as pending deprecation. (#5074) +* Mark `conda_build.conda_interface.linked_data` as pending deprecation. (#5074) +* Mark `conda_build.utils.linked_data_no_multichannels` as pending deprecation. (#5074) * Remove `conda_build.api.update_index`. (#5151) * Remove `conda_build.cli.main_build.main`. (#5151) * Remove `conda_build.cli.main_convert.main`. (#5151) @@ -77,6 +86,9 @@ ### Other * Remove unused Allure test report collection. (#5113) +* Removed dependency on `conda.plan`. (#5074) +* Removed almost all dependency on `conda.models.dist`. (#5074) +* Replaced usage of legacy `conda.models.dist.Dist` by `conda.models.records.PackageRecord`. (#5074) ### Contributors diff --git a/news/5074-dist-removal b/news/5074-dist-removal deleted file mode 100644 index 3d759fd06d..0000000000 --- a/news/5074-dist-removal +++ /dev/null @@ -1,35 +0,0 @@ -### Enhancements - -* - -### Bug fixes - -* - -### Deprecations - -* Removed `conda_build.conda_interface.get_installed_version`. (#5074) -* Removed `conda_build.environ.clean_pkg_cache`. (#5074) -* Removed `conda_build.inspect_pkg.dist_files`. (#5074) -* Removed `conda_build.inspect_pkg._installed`. (#5074) -* Removed `conda_build.post.dists_from_names`. (#5074) -* Removed `conda_build.post.FakeDist`. (#5074) -* Removed `conda_build.post._get_fake_pkg_dist`. (#5074) -* Mark `conda_build.conda_interface.Dist` as pending deprecation. (#5074) -* Mark `conda_build.conda_interface.display_actions` as pending deprecation. (#5074) -* Mark `conda_build.conda_interface.execute_actions` as pending deprecation. (#5074) -* Mark `conda_build.conda_interface.execute_plan` as pending deprecation. (#5074) -* Mark `conda_build.conda_interface.get_index` as pending deprecation. (#5074) -* Mark `conda_build.conda_interface.install_actions` as pending deprecation. (#5074) -* Mark `conda_build.conda_interface.linked` as pending deprecation. (#5074) -* Mark `conda_build.conda_interface.linked_data` as pending deprecation. (#5074) - -### Docs - -* - -### Other - -* Removed dependency on `conda.plan`. (#5074) -* Removed almost all dependency on `conda.models.dist`. (#5074) -* Replaced usage of legacy `conda.models.dist.Dist` by `conda.models.records.PackageRecord`. (#5074) From a569614c79e8f2b96124bb3b82d4f65aeea97509 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Fri, 26 Jan 2024 22:22:07 +0100 Subject: [PATCH 61/61] Apply style suggestions from review Signed-off-by: Marcel Bargull --- CHANGELOG.md | 6 +++--- conda_build/environ.py | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aac41a7ddc..5777412269 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,9 +86,9 @@ ### Other * Remove unused Allure test report collection. (#5113) -* Removed dependency on `conda.plan`. (#5074) -* Removed almost all dependency on `conda.models.dist`. (#5074) -* Replaced usage of legacy `conda.models.dist.Dist` by `conda.models.records.PackageRecord`. (#5074) +* Remove dependency on `conda.plan`. (#5074) +* Remove almost all dependency on `conda.models.dist`. (#5074) +* Replace usage of legacy `conda.models.dist.Dist` with `conda.models.records.PackageRecord`. (#5074) ### Contributors diff --git a/conda_build/environ.py b/conda_build/environ.py index 17ec237f30..c363588e3f 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -16,10 +16,7 @@ from logging import getLogger from os.path import join, normpath -from conda.base.constants import ( - DEFAULTS_CHANNEL_NAME, - UNKNOWN_CHANNEL, -) +from conda.base.constants import DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL from conda.common.io import env_vars from conda.core.index import LAST_CHANNEL_URLS from conda.core.link import PrefixSetup, UnlinkLinkTransaction