From be6bbc780c4d94b742b5872ecf2c97a2f38b398e Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 8 Nov 2023 11:53:24 +0100 Subject: [PATCH 01/24] add reproducer test --- tests/test_downstream.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_downstream.py b/tests/test_downstream.py index 959d44be..b8d6b86f 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -34,6 +34,22 @@ def test_build_recipes(): check_call(["conda-build", recipe], env=env) +def test_conda_build_with_aliased_channels(request): + "https://github.com/conda/conda-libmamba-solver/issues/363" + condarc = Path.home() / ".condarc" + if condarc.exists(): + condarc_contents = condarc.read_text() + request.addfinalizer(lambda: condarc.write_text(condarc_contents)) + else: + request.addfinalizer(lambda: condarc.unlink()) + condarc.write_text( + "channels: ['defaults']\n" + "default_channels: ['conda-forge']\n" + "solver: libmamba\n" + ) + check_call(["conda-build", DATA / "conda_build_recipes" / "jedi"]) + + def test_conda_lock(tmp_path): env = os.environ.copy() env["CONDA_SOLVER"] = "libmamba" From 615ebd6fe6699b50f9e59896d619fe0aa8dd421b Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 8 Nov 2023 13:17:13 +0100 Subject: [PATCH 02/24] move test --- tests/test_channels.py | 31 +++++++++++++++++++++++++++++-- tests/test_downstream.py | 16 ---------------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/tests/test_channels.py b/tests/test_channels.py index c3f6bae4..3637b8a4 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -18,6 +18,8 @@ from .utils import conda_subprocess, write_env_config +DATA = Path(__file__).parent / "data" + def test_channel_matchspec(): stdout, *_ = conda_inprocess( @@ -84,9 +86,19 @@ def test_channels_installed_unavailable(): assert retcode == 0 -def _setup_channels_alias(prefix): +def _setup_conda_forge_as_defaults(prefix, force=False): + write_env_config( + prefix, + force=force, + channels=["defaults"], + default_channels=["conda-forge"], + ) + + +def _setup_channels_alias(prefix, force=False): write_env_config( prefix, + force=force, channels=["conda-forge", "defaults"], channel_alias="https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud", migrated_channel_aliases=["https://conda.anaconda.org"], @@ -98,9 +110,10 @@ def _setup_channels_alias(prefix): ) -def _setup_channels_custom(prefix): +def _setup_channels_custom(prefix, force=False): write_env_config( prefix, + force=force, channels=["conda-forge", "defaults"], custom_channels={ "conda-forge": "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud", @@ -216,3 +229,17 @@ def test_encoding_file_paths(tmp_path: Path): print(process.stderr, file=sys.stderr) assert process.returncode == 0 assert list((tmp_path / "env" / "conda-meta").glob("test-package-*.json")) + + +def test_conda_build_with_aliased_channels(): + "https://github.com/conda/conda-libmamba-solver/issues/363" + condarc = Path.home() / ".condarc" + condarc_contents = condarc.read_text() if condarc.is_file() else None + try: + _setup_conda_forge_as_defaults(Path.home(), force=True) + conda_subprocess("build", DATA / "conda_build_recipes" / "jedi", "--channel=defaults", capture_output=False) + finally: + if condarc_contents: + condarc.write_text(condarc_contents) + else: + condarc.unlink() diff --git a/tests/test_downstream.py b/tests/test_downstream.py index b8d6b86f..959d44be 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -34,22 +34,6 @@ def test_build_recipes(): check_call(["conda-build", recipe], env=env) -def test_conda_build_with_aliased_channels(request): - "https://github.com/conda/conda-libmamba-solver/issues/363" - condarc = Path.home() / ".condarc" - if condarc.exists(): - condarc_contents = condarc.read_text() - request.addfinalizer(lambda: condarc.write_text(condarc_contents)) - else: - request.addfinalizer(lambda: condarc.unlink()) - condarc.write_text( - "channels: ['defaults']\n" - "default_channels: ['conda-forge']\n" - "solver: libmamba\n" - ) - check_call(["conda-build", DATA / "conda_build_recipes" / "jedi"]) - - def test_conda_lock(tmp_path): env = os.environ.copy() env["CONDA_SOLVER"] = "libmamba" From 00a3f783c6cbda326f0d54cad4248363cde76899 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 8 Nov 2023 13:17:26 +0100 Subject: [PATCH 03/24] use canonical channel name when available --- conda_libmamba_solver/solver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index 6b14203f..1a10107d 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -160,7 +160,6 @@ def solve_final_state( # From now on we _do_ require a solver and the index init_api_context() subdirs = self.subdirs - conda_bld_channels = () if self._called_from_conda_build(): log.info("Using solver via 'conda.plan.install_actions' (probably conda build)") # Problem: Conda build generates a custom index which happens to "forget" about @@ -179,6 +178,7 @@ def solve_final_state( IndexHelper = _CachedLibMambaIndexHelper else: IndexHelper = LibMambaIndexHelper + conda_bld_channels = () all_channels = [ *conda_bld_channels, @@ -880,7 +880,7 @@ def _package_record_from_json_payload( # Otherwise, these are records from the index kwargs["fn"] = pkg_filename - kwargs["channel"] = channel_info.channel + kwargs["channel"] = channel_info.channel.canonical_name or channel_info.channel kwargs["url"] = join_url(channel_info.full_url, pkg_filename) if not kwargs.get("subdir"): # missing in old channels kwargs["subdir"] = channel_info.channel.subdir From 1e0f3735541b90cba675c0db7b25f243054358e2 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 8 Nov 2023 13:19:14 +0100 Subject: [PATCH 04/24] pre-commit --- tests/test_channels.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_channels.py b/tests/test_channels.py index 3637b8a4..b01f5f91 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -237,7 +237,12 @@ def test_conda_build_with_aliased_channels(): condarc_contents = condarc.read_text() if condarc.is_file() else None try: _setup_conda_forge_as_defaults(Path.home(), force=True) - conda_subprocess("build", DATA / "conda_build_recipes" / "jedi", "--channel=defaults", capture_output=False) + conda_subprocess( + "build", + DATA / "conda_build_recipes" / "jedi", + "--channel=defaults", + capture_output=False, + ) finally: if condarc_contents: condarc.write_text(condarc_contents) From 45dea9e4f62a349cf0bc4e1bfab6d8b1e05b6394 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 8 Nov 2023 13:20:46 +0100 Subject: [PATCH 05/24] add news --- news/365-canonical-channel-names | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 news/365-canonical-channel-names diff --git a/news/365-canonical-channel-names b/news/365-canonical-channel-names new file mode 100644 index 00000000..58a9b4a1 --- /dev/null +++ b/news/365-canonical-channel-names @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Use canonical channel names (if available) in exported `PackageRecord` objects. Fixes an issue with conda-build and custom multichannels. (#363 via #365) + +### Deprecations + +* + +### Docs + +* + +### Other + +* From 487fa6b631d717305ed88c0e02d51413bb9e7217 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 8 Nov 2023 13:59:12 +0100 Subject: [PATCH 06/24] only for conda-build --- conda_libmamba_solver/solver.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index 1a10107d..61893d3c 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -42,7 +42,7 @@ SpecsConfigurationConflictError, UnsatisfiableError, ) -from conda.models.channel import Channel +from conda.models.channel import Channel, MultiChannel from conda.models.match_spec import MatchSpec from conda.models.records import PackageRecord, PrefixRecord from conda.models.version import VersionOrder @@ -880,7 +880,14 @@ def _package_record_from_json_payload( # Otherwise, these are records from the index kwargs["fn"] = pkg_filename - kwargs["channel"] = channel_info.channel.canonical_name or channel_info.channel + cname = channel_info.channel.canonical_name + multi_urls = context.custom_multichannels.get(cname) + if self._called_from_conda_build() and multi_urls: + # conda-build expects multichannel instances the Dist->PackageRecord mapping + # see https://github.com/conda/conda-libmamba-solver/issues/363 + kwargs["channel"] = MultiChannel(cname, multi_urls, channel_info.channel.platform) + else: + kwargs["channel"] = channel kwargs["url"] = join_url(channel_info.full_url, pkg_filename) if not kwargs.get("subdir"): # missing in old channels kwargs["subdir"] = channel_info.channel.subdir From c436dbd929c12e945c7b73c73d9f010cdf27613a Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 8 Nov 2023 17:24:40 +0100 Subject: [PATCH 07/24] parametrize recipes --- tests/test_downstream.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/test_downstream.py b/tests/test_downstream.py index 959d44be..124c8a0a 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -11,27 +11,30 @@ DATA = Path(__file__).parent / "data" -def test_build_recipes(): +@pytest.mark.parametrize( + "recipe", + [ + pytest.param(x, id=x.name) + for x in sorted((DATA / "conda_build_recipes").iterdir()) + if (x / "meta.yaml").is_file() + ] +) +def test_build_recipe(recipe): """ Adapted from https://github.com/mamba-org/boa/blob/3213180564/tests/test_mambabuild.py#L6 See /tests/data/conda_build_recipes/LICENSE for more details """ - recipes_dir = DATA / "conda_build_recipes" - - recipes = [str(x) for x in recipes_dir.iterdir() if x.is_dir()] + expected_fail_recipes = ["baddeps"] env = os.environ.copy() env["CONDA_SOLVER"] = "libmamba" - expected_fail_recipes = ["baddeps"] - for recipe in recipes: - recipe_name = Path(recipe).name - print(f"Running {recipe_name}") - if recipe_name in expected_fail_recipes: - with pytest.raises(CalledProcessError): - check_call(["conda-build", recipe], env=env) - else: + recipe_name = Path(recipe).name + if recipe_name in expected_fail_recipes: + with pytest.raises(CalledProcessError): check_call(["conda-build", recipe], env=env) + else: + check_call(["conda-build", recipe], env=env) def test_conda_lock(tmp_path): From d95a0e160bb00ba8b16681e7a7cdccc99be1ef4a Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 9 Nov 2023 10:45:15 +0100 Subject: [PATCH 08/24] try with just the string --- conda_libmamba_solver/solver.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index 61893d3c..9e41bc31 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -844,7 +844,8 @@ def _export_solved_records( # Fixes conda-build tests/test_api_build.py::test_croot_with_spaces if on_win and self._called_from_conda_build(): for record in out_state.records.values(): - record.channel.location = percent_decode(record.channel.location) + if record.channel.location: # multichannels like 'defaults' have no location + record.channel.location = percent_decode(record.channel.location) record.channel.name = percent_decode(record.channel.name) def _package_record_from_json_payload( @@ -881,11 +882,10 @@ def _package_record_from_json_payload( # Otherwise, these are records from the index kwargs["fn"] = pkg_filename cname = channel_info.channel.canonical_name - multi_urls = context.custom_multichannels.get(cname) - if self._called_from_conda_build() and multi_urls: - # conda-build expects multichannel instances the Dist->PackageRecord mapping + if self._called_from_conda_build() and cname in context.custom_multichannels: + # conda-build expects multichannel instances in the Dist->PackageRecord mapping # see https://github.com/conda/conda-libmamba-solver/issues/363 - kwargs["channel"] = MultiChannel(cname, multi_urls, channel_info.channel.platform) + kwargs["channel"] = cname else: kwargs["channel"] = channel kwargs["url"] = join_url(channel_info.full_url, pkg_filename) From 45658cd6bc8007533c70b2fde369d72e139c2b7f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:36:51 +0000 Subject: [PATCH 09/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_downstream.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_downstream.py b/tests/test_downstream.py index 124c8a0a..b97a4d51 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -12,12 +12,12 @@ @pytest.mark.parametrize( - "recipe", + "recipe", [ pytest.param(x, id=x.name) for x in sorted((DATA / "conda_build_recipes").iterdir()) if (x / "meta.yaml").is_file() - ] + ], ) def test_build_recipe(recipe): """ From e299c0b795e519a019edc6a40aacc51e3a5f46ad Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 9 Nov 2023 15:41:05 +0100 Subject: [PATCH 10/24] fix merge artifact --- tests/test_channels.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_channels.py b/tests/test_channels.py index 14da920f..b31bd4e8 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -243,6 +243,12 @@ def test_conda_build_with_aliased_channels(): DATA / "conda_build_recipes" / "jedi", "--channel=defaults", capture_output=False, + ) + finally: + if condarc_contents: + condarc.write_text(condarc_contents) + else: + condarc.unlink() def test_http_server_auth_none(http_server_auth_none): From 336646b004c67122663a66ab24fd2518421eb42a Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 9 Nov 2023 16:19:23 +0100 Subject: [PATCH 11/24] use the full channel, not just the name --- conda_libmamba_solver/solver.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index 9e41bc31..1948ad99 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -42,7 +42,7 @@ SpecsConfigurationConflictError, UnsatisfiableError, ) -from conda.models.channel import Channel, MultiChannel +from conda.models.channel import Channel from conda.models.match_spec import MatchSpec from conda.models.records import PackageRecord, PrefixRecord from conda.models.version import VersionOrder @@ -842,8 +842,10 @@ def _export_solved_records( ) # Fixes conda-build tests/test_api_build.py::test_croot_with_spaces - if on_win and self._called_from_conda_build(): + if self._called_from_conda_build(): for record in out_state.records.values(): + if "%" not in str(record): + continue if record.channel.location: # multichannels like 'defaults' have no location record.channel.location = percent_decode(record.channel.location) record.channel.name = percent_decode(record.channel.name) @@ -887,7 +889,7 @@ def _package_record_from_json_payload( # see https://github.com/conda/conda-libmamba-solver/issues/363 kwargs["channel"] = cname else: - kwargs["channel"] = channel + kwargs["channel"] = channel_info.channel kwargs["url"] = join_url(channel_info.full_url, pkg_filename) if not kwargs.get("subdir"): # missing in old channels kwargs["subdir"] = channel_info.channel.subdir From b188daa5aabe45dc783e3cb90f386606adc2c33f Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 9 Nov 2023 16:20:52 +0100 Subject: [PATCH 12/24] pre-commit --- conda_libmamba_solver/solver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index 1948ad99..dc6a5429 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -26,7 +26,6 @@ REPODATA_FN, UNKNOWN_CHANNEL, ChannelPriority, - on_win, ) from conda.base.context import context from conda.common.constants import NULL From 16532342e12aba8b056bd4211372ef7da35a4b67 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 9 Nov 2023 17:52:13 +0100 Subject: [PATCH 13/24] detect whether channel belongs to multichannel via url --- conda_libmamba_solver/solver.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index dc6a5429..ed019136 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -28,6 +28,7 @@ ChannelPriority, ) from conda.base.context import context +from conda.common.compat import on_win from conda.common.constants import NULL from conda.common.io import Spinner, timeout from conda.common.path import paths_equal @@ -841,7 +842,7 @@ def _export_solved_records( ) # Fixes conda-build tests/test_api_build.py::test_croot_with_spaces - if self._called_from_conda_build(): + if on_win and self._called_from_conda_build(): for record in out_state.records.values(): if "%" not in str(record): continue @@ -882,13 +883,15 @@ def _package_record_from_json_payload( # Otherwise, these are records from the index kwargs["fn"] = pkg_filename - cname = channel_info.channel.canonical_name - if self._called_from_conda_build() and cname in context.custom_multichannels: + kwargs["channel"] = channel_info.channel + if self._called_from_conda_build(): # conda-build expects multichannel instances in the Dist->PackageRecord mapping # see https://github.com/conda/conda-libmamba-solver/issues/363 - kwargs["channel"] = cname - else: - kwargs["channel"] = channel_info.channel + for multichannel_name, channels in context.custom_multichannels.items(): + urls = [url for c in channels for url in c.urls(with_credentials=False)] + if channel_info.noauth_url in urls: + kwargs["channel"] = multichannel_name + break kwargs["url"] = join_url(channel_info.full_url, pkg_filename) if not kwargs.get("subdir"): # missing in old channels kwargs["subdir"] = channel_info.channel.subdir From 44a6b49e5421abdcf7c83267d020ea8d85610521 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 11 Nov 2023 17:51:14 +0100 Subject: [PATCH 14/24] use separate CONDA_BLD_PATH dirs --- tests/test_channels.py | 5 ++++- tests/test_downstream.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_channels.py b/tests/test_channels.py index b31bd4e8..eef5bb76 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -232,10 +232,12 @@ def test_encoding_file_paths(tmp_path: Path): assert list((tmp_path / "env" / "conda-meta").glob("test-package-*.json")) -def test_conda_build_with_aliased_channels(): +def test_conda_build_with_aliased_channels(tmp_path): "https://github.com/conda/conda-libmamba-solver/issues/363" condarc = Path.home() / ".condarc" condarc_contents = condarc.read_text() if condarc.is_file() else None + env = os.environ.copy() + env["CONDA_BLD_PATH"] = str(tmp_path / "conda-bld") try: _setup_conda_forge_as_defaults(Path.home(), force=True) conda_subprocess( @@ -243,6 +245,7 @@ def test_conda_build_with_aliased_channels(): DATA / "conda_build_recipes" / "jedi", "--channel=defaults", capture_output=False, + env=env, ) finally: if condarc_contents: diff --git a/tests/test_downstream.py b/tests/test_downstream.py index b97a4d51..e6aaaaf3 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -19,7 +19,7 @@ if (x / "meta.yaml").is_file() ], ) -def test_build_recipe(recipe): +def test_build_recipe(recipe, tmp_path): """ Adapted from https://github.com/mamba-org/boa/blob/3213180564/tests/test_mambabuild.py#L6 @@ -29,6 +29,7 @@ def test_build_recipe(recipe): expected_fail_recipes = ["baddeps"] env = os.environ.copy() env["CONDA_SOLVER"] = "libmamba" + env["CONDA_BLD_PATH"] = str(tmp_path) recipe_name = Path(recipe).name if recipe_name in expected_fail_recipes: with pytest.raises(CalledProcessError): From 9c6773e6949683f657738132cb83e2d75906542a Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sun, 12 Nov 2023 18:13:54 +0100 Subject: [PATCH 15/24] only the channels test needs to be isolated --- tests/test_channels.py | 1 + tests/test_downstream.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_channels.py b/tests/test_channels.py index eef5bb76..0ee28865 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -243,6 +243,7 @@ def test_conda_build_with_aliased_channels(tmp_path): conda_subprocess( "build", DATA / "conda_build_recipes" / "jedi", + "--override-channels", "--channel=defaults", capture_output=False, env=env, diff --git a/tests/test_downstream.py b/tests/test_downstream.py index e6aaaaf3..b97a4d51 100644 --- a/tests/test_downstream.py +++ b/tests/test_downstream.py @@ -19,7 +19,7 @@ if (x / "meta.yaml").is_file() ], ) -def test_build_recipe(recipe, tmp_path): +def test_build_recipe(recipe): """ Adapted from https://github.com/mamba-org/boa/blob/3213180564/tests/test_mambabuild.py#L6 @@ -29,7 +29,6 @@ def test_build_recipe(recipe, tmp_path): expected_fail_recipes = ["baddeps"] env = os.environ.copy() env["CONDA_SOLVER"] = "libmamba" - env["CONDA_BLD_PATH"] = str(tmp_path) recipe_name = Path(recipe).name if recipe_name in expected_fail_recipes: with pytest.raises(CalledProcessError): From cbef76c828a0948bba3d5949678e7b9ff7c8c847 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sun, 12 Nov 2023 18:14:42 +0100 Subject: [PATCH 16/24] cache this called_from_conda_build call a bit --- conda_libmamba_solver/solver.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index ed019136..b23ab515 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -820,8 +820,9 @@ def _export_solved_records( else: log.warn("Tried to unlink %s but it is not installed or manageable?", filename) + for_conda_build = self._called_from_conda_build() for channel, filename, json_payload in to_link: - record = self._package_record_from_json_payload(index, channel, filename, json_payload) + record = self._package_record_from_json_payload(index, channel, filename, json_payload, for_conda_build=for_conda_build) # We need this check below to make sure noarch package get reinstalled # record metadata coming from libmamba is incomplete and won't pass the # noarch checks -- to fix it, we swap the metadata-only record with its locally @@ -842,7 +843,7 @@ def _export_solved_records( ) # Fixes conda-build tests/test_api_build.py::test_croot_with_spaces - if on_win and self._called_from_conda_build(): + if on_win and for_conda_build: for record in out_state.records.values(): if "%" not in str(record): continue @@ -851,14 +852,15 @@ def _export_solved_records( record.channel.name = percent_decode(record.channel.name) def _package_record_from_json_payload( - self, index: LibMambaIndexHelper, channel: str, pkg_filename: str, json_payload: str + self, index: LibMambaIndexHelper, channel: str, pkg_filename: str, json_payload: str, + for_conda_build: bool = False, ) -> PackageRecord: """ The libmamba transactions cannot return full-blown objects from the C/C++ side. Instead, it returns the instructions to build one on the Python side: channel_info: dict - Channel data, as built in .index.LibmambaIndexHelper._fetch_channel() + Channel datas, as built in .index.LibmambaIndexHelper._fetch_channel() This is retrieved from the .index._index mapping, keyed by channel URLs pkg_filename: str The filename (.tar.bz2 or .conda) of the selected record. @@ -884,11 +886,11 @@ def _package_record_from_json_payload( # Otherwise, these are records from the index kwargs["fn"] = pkg_filename kwargs["channel"] = channel_info.channel - if self._called_from_conda_build(): + if for_conda_build: # conda-build expects multichannel instances in the Dist->PackageRecord mapping # see https://github.com/conda/conda-libmamba-solver/issues/363 - for multichannel_name, channels in context.custom_multichannels.items(): - urls = [url for c in channels for url in c.urls(with_credentials=False)] + for multichannel_name, mc_channels in context.custom_multichannels.items(): + urls = [url for c in mc_channels for url in c.urls(with_credentials=False)] if channel_info.noauth_url in urls: kwargs["channel"] = multichannel_name break From 8d9d6c03fd13e6a86a16fc396e1f7413baa269ea Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sun, 12 Nov 2023 18:53:58 +0100 Subject: [PATCH 17/24] make sure we clear the repo before reloading it --- conda_libmamba_solver/index.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index 3e19905b..c3e951a4 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -155,10 +155,12 @@ def reload_local_channels(self): Reload a channel that was previously loaded from a local directory. """ for noauth_url, info in self._index.items(): - if noauth_url.startswith("file://"): + if noauth_url.startswith("file://") or info.channel.scheme == "file": url, json_path = self._fetch_channel(info.full_url) - new = self._json_path_to_repo_info(url, json_path) - self._repos[self._repos.index(info.repo)] = new.repo + repo_position = self._repos.index(info.repo) + info.repo.clear(True) + new = self._json_path_to_repo_info(url, json_path, try_solv=False) + self._repos[repo_position] = new.repo self._index[noauth_url] = new set_channel_priorities(self._index) @@ -234,20 +236,24 @@ def _fetch_channel(self, url: str) -> Tuple[str, os.PathLike]: return url, json_path - def _json_path_to_repo_info(self, url: str, json_path: str) -> Optional[_ChannelRepoInfo]: + def _json_path_to_repo_info(self, url: str, json_path: str, try_solv: bool = False) -> Optional[_ChannelRepoInfo]: channel = Channel.from_url(url) noauth_url = channel.urls(with_credentials=False, subdirs=(channel.subdir,))[0] json_path = Path(json_path) - solv_path = json_path.parent / f"{json_path.stem}.solv" try: json_stat = json_path.stat() except OSError as exc: log.debug("Failed to stat %s", json_path, exc_info=exc) json_stat = None - try: - solv_stat = solv_path.stat() - except OSError as exc: - log.debug("Failed to stat %s", solv_path, exc_info=exc) + if try_solv: + try: + solv_path = json_path.parent / f"{json_path.stem}.solv" + solv_stat = solv_path.stat() + except OSError as exc: + log.debug("Failed to stat %s", solv_path, exc_info=exc) + solv_stat = None + else: + solv_path = None solv_stat = None if solv_stat is None and json_stat is None: From cf3f2c8833b8cd09e59e42c0696d23d158acf9fd Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 13 Nov 2023 10:47:44 +0100 Subject: [PATCH 18/24] try shorter env path on windows --- tests/test_channels.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_channels.py b/tests/test_channels.py index 0ee28865..f3820f01 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -9,7 +9,7 @@ import pytest from conda.base.context import reset_context -from conda.common.compat import on_linux +from conda.common.compat import on_linux, on_win from conda.common.io import env_vars from conda.core.prefix_data import PrefixData from conda.models.channel import Channel @@ -237,7 +237,10 @@ def test_conda_build_with_aliased_channels(tmp_path): condarc = Path.home() / ".condarc" condarc_contents = condarc.read_text() if condarc.is_file() else None env = os.environ.copy() - env["CONDA_BLD_PATH"] = str(tmp_path / "conda-bld") + if on_win: + env["CONDA_BLD_PATH"] = str(Path(os.environ.get("RUNNER_TEMP", tmp_path), "bld")) + else: + env["CONDA_BLD_PATH"] = str(tmp_path / "conda-bld") try: _setup_conda_forge_as_defaults(Path.home(), force=True) conda_subprocess( From 9aaa8b48c2d7634b2a00bda176733225c4b5d6cb Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 13 Nov 2023 10:52:23 +0100 Subject: [PATCH 19/24] use restricted unicode on windows --- tests/channel_testing/helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/channel_testing/helpers.py b/tests/channel_testing/helpers.py index 549e05f7..8318cd3d 100644 --- a/tests/channel_testing/helpers.py +++ b/tests/channel_testing/helpers.py @@ -10,6 +10,7 @@ from typing import Tuple import pytest +from conda.common.compat import on_win from conda.testing.integration import _get_temp_prefix, run_command from xprocess import ProcessStarter @@ -141,7 +142,7 @@ def create_with_channel( "conda", "create", "-p", - _get_temp_prefix(), + _get_temp_prefix(use_restricted_unicode=on_win), f"--solver={solver}", "--json", "--override-channels", @@ -157,7 +158,7 @@ def create_with_channel( def create_with_channel_in_process(channel, solver="libmamba", **kwargs) -> Tuple[str, str, int]: stdout, stderr, returncode = run_command( "create", - _get_temp_prefix(), + _get_temp_prefix(use_restricted_unicode=on_win), f"--solver={solver}", "--json", "--override-channels", From f7bda4a6d41a5c4476f855fd7ad276912ff8ea70 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 13 Nov 2023 10:53:22 +0100 Subject: [PATCH 20/24] pre-commit --- conda_libmamba_solver/index.py | 6 ++++-- conda_libmamba_solver/solver.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index c3e951a4..edcc6824 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -236,7 +236,9 @@ def _fetch_channel(self, url: str) -> Tuple[str, os.PathLike]: return url, json_path - def _json_path_to_repo_info(self, url: str, json_path: str, try_solv: bool = False) -> Optional[_ChannelRepoInfo]: + def _json_path_to_repo_info( + self, url: str, json_path: str, try_solv: bool = False + ) -> Optional[_ChannelRepoInfo]: channel = Channel.from_url(url) noauth_url = channel.urls(with_credentials=False, subdirs=(channel.subdir,))[0] json_path = Path(json_path) @@ -247,7 +249,7 @@ def _json_path_to_repo_info(self, url: str, json_path: str, try_solv: bool = Fal json_stat = None if try_solv: try: - solv_path = json_path.parent / f"{json_path.stem}.solv" + solv_path = json_path.parent / f"{json_path.stem}.solv" solv_stat = solv_path.stat() except OSError as exc: log.debug("Failed to stat %s", solv_path, exc_info=exc) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index b23ab515..2d307ce2 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -822,7 +822,9 @@ def _export_solved_records( for_conda_build = self._called_from_conda_build() for channel, filename, json_payload in to_link: - record = self._package_record_from_json_payload(index, channel, filename, json_payload, for_conda_build=for_conda_build) + record = self._package_record_from_json_payload( + index, channel, filename, json_payload, for_conda_build=for_conda_build + ) # We need this check below to make sure noarch package get reinstalled # record metadata coming from libmamba is incomplete and won't pass the # noarch checks -- to fix it, we swap the metadata-only record with its locally @@ -852,7 +854,11 @@ def _export_solved_records( record.channel.name = percent_decode(record.channel.name) def _package_record_from_json_payload( - self, index: LibMambaIndexHelper, channel: str, pkg_filename: str, json_payload: str, + self, + index: LibMambaIndexHelper, + channel: str, + pkg_filename: str, + json_payload: str, for_conda_build: bool = False, ) -> PackageRecord: """ From 1da19fefeea1deb06612345213b6cb86ccfcdcb4 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 13 Nov 2023 11:28:55 +0100 Subject: [PATCH 21/24] unicode restrictions on this test only --- tests/channel_testing/helpers.py | 5 ++--- tests/test_channels.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/channel_testing/helpers.py b/tests/channel_testing/helpers.py index 8318cd3d..549e05f7 100644 --- a/tests/channel_testing/helpers.py +++ b/tests/channel_testing/helpers.py @@ -10,7 +10,6 @@ from typing import Tuple import pytest -from conda.common.compat import on_win from conda.testing.integration import _get_temp_prefix, run_command from xprocess import ProcessStarter @@ -142,7 +141,7 @@ def create_with_channel( "conda", "create", "-p", - _get_temp_prefix(use_restricted_unicode=on_win), + _get_temp_prefix(), f"--solver={solver}", "--json", "--override-channels", @@ -158,7 +157,7 @@ def create_with_channel( def create_with_channel_in_process(channel, solver="libmamba", **kwargs) -> Tuple[str, str, int]: stdout, stderr, returncode = run_command( "create", - _get_temp_prefix(use_restricted_unicode=on_win), + _get_temp_prefix(), f"--solver={solver}", "--json", "--override-channels", diff --git a/tests/test_channels.py b/tests/test_channels.py index f3820f01..ed296608 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -289,7 +289,7 @@ def test_http_server_auth_token_in_defaults(http_server_auth_token): conda_subprocess( "create", "-p", - _get_temp_prefix(), + _get_temp_prefix(use_restricted_unicode=on_win), "--solver=libmamba", "test-package", ) From cee4693266b1d054bd85c2667140e7d712bbf20d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 13 Nov 2023 16:14:28 +0100 Subject: [PATCH 22/24] extend stackvana example --- .../conda_build_recipes/stackvana/meta.yaml | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/data/conda_build_recipes/stackvana/meta.yaml b/tests/data/conda_build_recipes/stackvana/meta.yaml index 24286527..a6fb8e0f 100644 --- a/tests/data/conda_build_recipes/stackvana/meta.yaml +++ b/tests/data/conda_build_recipes/stackvana/meta.yaml @@ -1,5 +1,6 @@ {% set name = "stackvana-core" %} {% set version = "0.2021.43" %} +{% set eups_product = "lsst_distrib" %} package: name: {{ name|lower }} @@ -15,11 +16,45 @@ outputs: script: - echo "BUILDING IMPL" >> $PREFIX/stackvana-core-impl # [unix] - echo "BUILDING IMPL" >> %PREFIX%/stackvana-core-impl # [win] + test: + commands: + - echo OK - name: stackvana-core version: {{ version }} - run_exports: - - {{ pin_subpackage('stackvana-core-impl', exact=True) }} - + build: + script: + - echo "BUILDING CORE" >> $PREFIX/stackvana-core # [unix] + - echo "BUILDING CORE" >> %PREFIX%/stackvana-core # [win] + run_exports: + - {{ pin_subpackage('stackvana-core-impl', exact=True) }} requirements: run: - {{ pin_subpackage('stackvana-core-impl', exact=True) }} + test: + commands: + - echo OK + - name: stackvana-{{ eups_product }} + version: {{ version }} + build: + script: + - echo "BUILDING {{ eups_product }}" >> $PREFIX/stackvana-{{ eups_product }} # [unix] + - echo "BUILDING {{ eups_product }}" >> %PREFIX%/stackvana-{{ eups_product }} # [win] + requirements: + host: + - stackvana-core =={{ version }} + run: + - stackvana-core =={{ version }} + test: + commands: + - echo OK + - name: stackvana + version: {{ version }} + build: + script: + - echo "BUILDING STACKVANA" >> $PREFIX/stackvana # [unix] + - echo "BUILDING STACKVANA" >> %PREFIX%/stackvana # [win] + requirements: + - {{ pin_subpackage("stackvana-" ~ eups_product, max_pin="x.x.x") }} + test: + commands: + - echo OK From 093b3e012711dc1405346d459b4627d02c8320ad Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 13 Nov 2023 16:45:25 +0100 Subject: [PATCH 23/24] pre-commit [ci skip] --- tests/data/conda_build_recipes/stackvana/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/conda_build_recipes/stackvana/meta.yaml b/tests/data/conda_build_recipes/stackvana/meta.yaml index a6fb8e0f..2991ce80 100644 --- a/tests/data/conda_build_recipes/stackvana/meta.yaml +++ b/tests/data/conda_build_recipes/stackvana/meta.yaml @@ -32,7 +32,7 @@ outputs: - {{ pin_subpackage('stackvana-core-impl', exact=True) }} test: commands: - - echo OK + - echo OK - name: stackvana-{{ eups_product }} version: {{ version }} build: From a576cfb538d8af466f692f45924f9d3de010ef42 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 13 Nov 2023 16:46:49 +0100 Subject: [PATCH 24/24] retrigger