diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index 3e19905b..edcc6824 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -155,10 +155,12 @@ def reload_local_channels(self): Reload a channel that was previously loaded from a local directory. """ for noauth_url, info in self._index.items(): - if noauth_url.startswith("file://"): + if noauth_url.startswith("file://") or info.channel.scheme == "file": url, json_path = self._fetch_channel(info.full_url) - new = self._json_path_to_repo_info(url, json_path) - self._repos[self._repos.index(info.repo)] = new.repo + repo_position = self._repos.index(info.repo) + info.repo.clear(True) + new = self._json_path_to_repo_info(url, json_path, try_solv=False) + self._repos[repo_position] = new.repo self._index[noauth_url] = new set_channel_priorities(self._index) @@ -234,20 +236,26 @@ def _fetch_channel(self, url: str) -> Tuple[str, os.PathLike]: return url, json_path - def _json_path_to_repo_info(self, url: str, json_path: str) -> Optional[_ChannelRepoInfo]: + def _json_path_to_repo_info( + self, url: str, json_path: str, try_solv: bool = False + ) -> Optional[_ChannelRepoInfo]: channel = Channel.from_url(url) noauth_url = channel.urls(with_credentials=False, subdirs=(channel.subdir,))[0] json_path = Path(json_path) - solv_path = json_path.parent / f"{json_path.stem}.solv" try: json_stat = json_path.stat() except OSError as exc: log.debug("Failed to stat %s", json_path, exc_info=exc) json_stat = None - try: - solv_stat = solv_path.stat() - except OSError as exc: - log.debug("Failed to stat %s", solv_path, exc_info=exc) + if try_solv: + try: + solv_path = json_path.parent / f"{json_path.stem}.solv" + solv_stat = solv_path.stat() + except OSError as exc: + log.debug("Failed to stat %s", solv_path, exc_info=exc) + solv_stat = None + else: + solv_path = None solv_stat = None if solv_stat is None and json_stat is None: diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index 98f78825..d52bc713 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -26,9 +26,9 @@ REPODATA_FN, UNKNOWN_CHANNEL, ChannelPriority, - on_win, ) from conda.base.context import context +from conda.common.compat import on_win from conda.common.constants import NULL from conda.common.io import Spinner, timeout from conda.common.path import paths_equal @@ -160,7 +160,6 @@ def solve_final_state( # From now on we _do_ require a solver and the index init_api_context() subdirs = self.subdirs - conda_bld_channels = () if self._called_from_conda_build(): log.info("Using solver via 'conda.plan.install_actions' (probably conda build)") # Problem: Conda build generates a custom index which happens to "forget" about @@ -179,6 +178,7 @@ def solve_final_state( IndexHelper = _CachedLibMambaIndexHelper else: IndexHelper = LibMambaIndexHelper + conda_bld_channels = () all_channels = [ *conda_bld_channels, @@ -826,8 +826,11 @@ def _export_solved_records( else: log.warn("Tried to unlink %s but it is not installed or manageable?", filename) + for_conda_build = self._called_from_conda_build() for channel, filename, json_payload in to_link: - record = self._package_record_from_json_payload(index, channel, filename, json_payload) + record = self._package_record_from_json_payload( + index, channel, filename, json_payload, for_conda_build=for_conda_build + ) # We need this check below to make sure noarch package get reinstalled # record metadata coming from libmamba is incomplete and won't pass the # noarch checks -- to fix it, we swap the metadata-only record with its locally @@ -848,20 +851,28 @@ def _export_solved_records( ) # Fixes conda-build tests/test_api_build.py::test_croot_with_spaces - if on_win and self._called_from_conda_build(): + if on_win and for_conda_build: for record in out_state.records.values(): - record.channel.location = percent_decode(record.channel.location) + if "%" not in str(record): + continue + if record.channel.location: # multichannels like 'defaults' have no location + record.channel.location = percent_decode(record.channel.location) record.channel.name = percent_decode(record.channel.name) def _package_record_from_json_payload( - self, index: LibMambaIndexHelper, channel: str, pkg_filename: str, json_payload: str + self, + index: LibMambaIndexHelper, + channel: str, + pkg_filename: str, + json_payload: str, + for_conda_build: bool = False, ) -> PackageRecord: """ The libmamba transactions cannot return full-blown objects from the C/C++ side. Instead, it returns the instructions to build one on the Python side: channel_info: dict - Channel data, as built in .index.LibmambaIndexHelper._fetch_channel() + Channel datas, as built in .index.LibmambaIndexHelper._fetch_channel() This is retrieved from the .index._index mapping, keyed by channel URLs pkg_filename: str The filename (.tar.bz2 or .conda) of the selected record. @@ -887,6 +898,14 @@ def _package_record_from_json_payload( # Otherwise, these are records from the index kwargs["fn"] = pkg_filename kwargs["channel"] = channel_info.channel + if for_conda_build: + # conda-build expects multichannel instances in the Dist->PackageRecord mapping + # see https://github.com/conda/conda-libmamba-solver/issues/363 + for multichannel_name, mc_channels in context.custom_multichannels.items(): + urls = [url for c in mc_channels for url in c.urls(with_credentials=False)] + if channel_info.noauth_url in urls: + kwargs["channel"] = multichannel_name + break kwargs["url"] = join_url(channel_info.full_url, pkg_filename) if not kwargs.get("subdir"): # missing in old channels kwargs["subdir"] = channel_info.channel.subdir diff --git a/news/365-canonical-channel-names b/news/365-canonical-channel-names new file mode 100644 index 00000000..58a9b4a1 --- /dev/null +++ b/news/365-canonical-channel-names @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Use canonical channel names (if available) in exported `PackageRecord` objects. Fixes an issue with conda-build and custom multichannels. (#363 via #365) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/data/conda_build_recipes/stackvana/meta.yaml b/tests/data/conda_build_recipes/stackvana/meta.yaml index 24286527..2991ce80 100644 --- a/tests/data/conda_build_recipes/stackvana/meta.yaml +++ b/tests/data/conda_build_recipes/stackvana/meta.yaml @@ -1,5 +1,6 @@ {% set name = "stackvana-core" %} {% set version = "0.2021.43" %} +{% set eups_product = "lsst_distrib" %} package: name: {{ name|lower }} @@ -15,11 +16,45 @@ outputs: script: - echo "BUILDING IMPL" >> $PREFIX/stackvana-core-impl # [unix] - echo "BUILDING IMPL" >> %PREFIX%/stackvana-core-impl # [win] + test: + commands: + - echo OK - name: stackvana-core version: {{ version }} - run_exports: - - {{ pin_subpackage('stackvana-core-impl', exact=True) }} - + build: + script: + - echo "BUILDING CORE" >> $PREFIX/stackvana-core # [unix] + - echo "BUILDING CORE" >> %PREFIX%/stackvana-core # [win] + run_exports: + - {{ pin_subpackage('stackvana-core-impl', exact=True) }} requirements: run: - {{ pin_subpackage('stackvana-core-impl', exact=True) }} + test: + commands: + - echo OK + - name: stackvana-{{ eups_product }} + version: {{ version }} + build: + script: + - echo "BUILDING {{ eups_product }}" >> $PREFIX/stackvana-{{ eups_product }} # [unix] + - echo "BUILDING {{ eups_product }}" >> %PREFIX%/stackvana-{{ eups_product }} # [win] + requirements: + host: + - stackvana-core =={{ version }} + run: + - stackvana-core =={{ version }} + test: + commands: + - echo OK + - name: stackvana + version: {{ version }} + build: + script: + - echo "BUILDING STACKVANA" >> $PREFIX/stackvana # [unix] + - echo "BUILDING STACKVANA" >> %PREFIX%/stackvana # [win] + requirements: + - {{ pin_subpackage("stackvana-" ~ eups_product, max_pin="x.x.x") }} + test: + commands: + - echo OK diff --git a/tests/test_channels.py b/tests/test_channels.py index 94aba60b..ed296608 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -9,7 +9,7 @@ import pytest from conda.base.context import reset_context -from conda.common.compat import on_linux +from conda.common.compat import on_linux, on_win from conda.common.io import env_vars from conda.core.prefix_data import PrefixData from conda.models.channel import Channel @@ -23,6 +23,8 @@ from .channel_testing.helpers import create_with_channel from .utils import conda_subprocess, write_env_config +DATA = Path(__file__).parent / "data" + def test_channel_matchspec(): stdout, *_ = conda_inprocess( @@ -89,9 +91,19 @@ def test_channels_installed_unavailable(): assert retcode == 0 -def _setup_channels_alias(prefix): +def _setup_conda_forge_as_defaults(prefix, force=False): + write_env_config( + prefix, + force=force, + channels=["defaults"], + default_channels=["conda-forge"], + ) + + +def _setup_channels_alias(prefix, force=False): write_env_config( prefix, + force=force, channels=["conda-forge", "defaults"], channel_alias="https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud", migrated_channel_aliases=["https://conda.anaconda.org"], @@ -103,9 +115,10 @@ def _setup_channels_alias(prefix): ) -def _setup_channels_custom(prefix): +def _setup_channels_custom(prefix, force=False): write_env_config( prefix, + force=force, channels=["conda-forge", "defaults"], custom_channels={ "conda-forge": "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud", @@ -219,6 +232,32 @@ def test_encoding_file_paths(tmp_path: Path): assert list((tmp_path / "env" / "conda-meta").glob("test-package-*.json")) +def test_conda_build_with_aliased_channels(tmp_path): + "https://github.com/conda/conda-libmamba-solver/issues/363" + condarc = Path.home() / ".condarc" + condarc_contents = condarc.read_text() if condarc.is_file() else None + env = os.environ.copy() + if on_win: + env["CONDA_BLD_PATH"] = str(Path(os.environ.get("RUNNER_TEMP", tmp_path), "bld")) + else: + env["CONDA_BLD_PATH"] = str(tmp_path / "conda-bld") + try: + _setup_conda_forge_as_defaults(Path.home(), force=True) + conda_subprocess( + "build", + DATA / "conda_build_recipes" / "jedi", + "--override-channels", + "--channel=defaults", + capture_output=False, + env=env, + ) + finally: + if condarc_contents: + condarc.write_text(condarc_contents) + else: + condarc.unlink() + + def test_http_server_auth_none(http_server_auth_none): create_with_channel(http_server_auth_none) @@ -247,12 +286,12 @@ def test_http_server_auth_token_in_defaults(http_server_auth_token): ) reset_context() conda_subprocess("info", capture_output=False) - conda_inprocess( + conda_subprocess( "create", - _get_temp_prefix(), + "-p", + _get_temp_prefix(use_restricted_unicode=on_win), "--solver=libmamba", "test-package", - no_capture=True, ) finally: if condarc_contents: diff --git a/tests/test_downstream.py b/tests/test_downstream.py index 959d44be..b97a4d51 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -11,27 +11,30 @@ DATA = Path(__file__).parent / "data" -def test_build_recipes(): +@pytest.mark.parametrize( + "recipe", + [ + pytest.param(x, id=x.name) + for x in sorted((DATA / "conda_build_recipes").iterdir()) + if (x / "meta.yaml").is_file() + ], +) +def test_build_recipe(recipe): """ Adapted from https://github.com/mamba-org/boa/blob/3213180564/tests/test_mambabuild.py#L6 See /tests/data/conda_build_recipes/LICENSE for more details """ - recipes_dir = DATA / "conda_build_recipes" - - recipes = [str(x) for x in recipes_dir.iterdir() if x.is_dir()] + expected_fail_recipes = ["baddeps"] env = os.environ.copy() env["CONDA_SOLVER"] = "libmamba" - expected_fail_recipes = ["baddeps"] - for recipe in recipes: - recipe_name = Path(recipe).name - print(f"Running {recipe_name}") - if recipe_name in expected_fail_recipes: - with pytest.raises(CalledProcessError): - check_call(["conda-build", recipe], env=env) - else: + recipe_name = Path(recipe).name + if recipe_name in expected_fail_recipes: + with pytest.raises(CalledProcessError): check_call(["conda-build", recipe], env=env) + else: + check_call(["conda-build", recipe], env=env) def test_conda_lock(tmp_path):