From 9586f4f259a4105a2fda47b3f3b2085d9cd8b878 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 22 Nov 2023 12:28:03 +0100 Subject: [PATCH 1/7] use index in offline mode when called from conda-build --- conda_libmamba_solver/index.py | 7 +++++++ conda_libmamba_solver/solver.py | 4 ++-- tests/data/conda_build_recipes/stackvana/meta.yaml | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index edcc6824..84c03111 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -409,3 +409,10 @@ def _process_query_result( # for conda-build _CachedLibMambaIndexHelper = lru_cache(maxsize=None)(LibMambaIndexHelper) + + +# conda-build needs to operate offline for the index +def _LibMambaIndexHelperOffline(*args, **kwargs): + with context._override("offline", True): + helper = _CachedLibMambaIndexHelper(*args, **kwargs) + return helper diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index d52bc713..821adee0 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -49,7 +49,7 @@ from . import __version__ from .exceptions import LibMambaUnsatisfiableError -from .index import LibMambaIndexHelper, _CachedLibMambaIndexHelper +from .index import LibMambaIndexHelper, _LibMambaIndexHelperOffline from .mamba_utils import init_api_context, mamba_version from .state import SolverInputState, SolverOutputState from .utils import is_channel_available @@ -175,7 +175,7 @@ def solve_final_state( rec.channel: None for rec in self._index if rec.channel.scheme == "file" } # Cache indices for conda-build, it gets heavy otherwise - IndexHelper = _CachedLibMambaIndexHelper + IndexHelper = _LibMambaIndexHelperOffline else: IndexHelper = LibMambaIndexHelper conda_bld_channels = () diff --git a/tests/data/conda_build_recipes/stackvana/meta.yaml b/tests/data/conda_build_recipes/stackvana/meta.yaml index 2991ce80..b062368a 100644 --- a/tests/data/conda_build_recipes/stackvana/meta.yaml +++ b/tests/data/conda_build_recipes/stackvana/meta.yaml @@ -1,4 +1,4 @@ -{% set name = "stackvana-core" %} +{% set name = "stackvana-split" %} {% set version = "0.2021.43" %} {% set eups_product = "lsst_distrib" %} From 1bb95d0c4f5432dac7cefd860fa6a44d1186d97b Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 23 Nov 2023 10:06:43 +0100 Subject: [PATCH 2/7] add docstring --- conda_libmamba_solver/index.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index 84c03111..fd819d7c 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -411,8 +411,12 @@ def _process_query_result( _CachedLibMambaIndexHelper = lru_cache(maxsize=None)(LibMambaIndexHelper) -# conda-build needs to operate offline for the index def _LibMambaIndexHelperOffline(*args, **kwargs): + """ + conda-build needs to operate offline so the index doesn't get updated + accidentally during long build phases. + + See https://github.com/conda/conda-libmamba-solver/issues/386 + """ with context._override("offline", True): - helper = _CachedLibMambaIndexHelper(*args, **kwargs) - return helper + return _CachedLibMambaIndexHelper(*args, **kwargs) From 82ff57c5eb58d6b761ed1d22de3b4a747d36f708 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 23 Nov 2023 10:16:47 +0100 Subject: [PATCH 3/7] add warning in old conda versions --- conda_libmamba_solver/index.py | 22 ++++++++++++++++++---- conda_libmamba_solver/solver.py | 4 ++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index fd819d7c..06153281 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -79,6 +79,7 @@ from typing import Dict, Iterable, Optional, Tuple, Union import libmambapy as api +from conda import __version__ as conda_version from conda.base.constants import REPODATA_FN from conda.base.context import context from conda.common.io import DummyExecutor, ThreadLimitedThreadPoolExecutor @@ -88,6 +89,7 @@ from conda.models.channel import Channel from conda.models.match_spec import MatchSpec from conda.models.records import PackageRecord +from conda.models.version import VersionOrder from .mamba_utils import set_channel_priorities from .state import IndexHelper @@ -411,12 +413,24 @@ def _process_query_result( _CachedLibMambaIndexHelper = lru_cache(maxsize=None)(LibMambaIndexHelper) -def _LibMambaIndexHelperOffline(*args, **kwargs): +def _LibMambaIndexForCondaBuild(*args, **kwargs): """ - conda-build needs to operate offline so the index doesn't get updated - accidentally during long build phases. - See https://github.com/conda/conda-libmamba-solver/issues/386 + + conda-build needs to operate offline so the index doesn't get updated + accidentally during long build phases. However, this is only guaranteed + to work if https://github.com/conda/conda/pull/13357 is applied. Otherwise + the condarc configuration might be ignored, resulting in bad index configuration + and missing packages anyway. """ + if VersionOrder(conda_version) <= VersionOrder("23.10.0"): + log.warning( + "conda-build requires conda >=23.11.0 for offline index support. " + "Falling back to online index. This might result in KeyError messages, " + "specially if the remote repodata is updated during the build phase. " + "See https://github.com/conda/conda-libmamba-solver/issues/386." + ) + return _CachedLibMambaIndexHelper(*args, **kwargs) + with context._override("offline", True): return _CachedLibMambaIndexHelper(*args, **kwargs) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index 821adee0..85ebb9a8 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -49,7 +49,7 @@ from . import __version__ from .exceptions import LibMambaUnsatisfiableError -from .index import LibMambaIndexHelper, _LibMambaIndexHelperOffline +from .index import LibMambaIndexHelper, _LibMambaIndexForCondaBuild from .mamba_utils import init_api_context, mamba_version from .state import SolverInputState, SolverOutputState from .utils import is_channel_available @@ -175,7 +175,7 @@ def solve_final_state( rec.channel: None for rec in self._index if rec.channel.scheme == "file" } # Cache indices for conda-build, it gets heavy otherwise - IndexHelper = _LibMambaIndexHelperOffline + IndexHelper = _LibMambaIndexForCondaBuild else: IndexHelper = LibMambaIndexHelper conda_bld_channels = () From 2ff22c72f868ede1af99d96aa3f2ea979b2b644c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 23 Nov 2023 10:16:51 +0100 Subject: [PATCH 4/7] add news --- news/395-offline-conda-build | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 news/395-offline-conda-build diff --git a/news/395-offline-conda-build b/news/395-offline-conda-build new file mode 100644 index 00000000..f54e5c16 --- /dev/null +++ b/news/395-offline-conda-build @@ -0,0 +1,20 @@ +### Enhancements + +* + +### Bug fixes + +* Instantiate `IndexHelper` in offline mode for compatibility with conda-build. Otherwise + the index can get out of sync during long build processes. (#386 via #395) + +### Deprecations + +* + +### Docs + +* + +### Other + +* From 395d538296b64a34927e5ba8f0facc5bf5c0161c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 23 Nov 2023 10:36:35 +0100 Subject: [PATCH 5/7] pre-commit --- conda_libmamba_solver/index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index 06153281..f0c8b756 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -416,7 +416,7 @@ def _process_query_result( def _LibMambaIndexForCondaBuild(*args, **kwargs): """ See https://github.com/conda/conda-libmamba-solver/issues/386 - + conda-build needs to operate offline so the index doesn't get updated accidentally during long build phases. However, this is only guaranteed to work if https://github.com/conda/conda/pull/13357 is applied. Otherwise @@ -431,6 +431,6 @@ def _LibMambaIndexForCondaBuild(*args, **kwargs): "See https://github.com/conda/conda-libmamba-solver/issues/386." ) return _CachedLibMambaIndexHelper(*args, **kwargs) - + with context._override("offline", True): return _CachedLibMambaIndexHelper(*args, **kwargs) From db269e95e14a28df4de1d5aa4acd10a0bff2af54 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 5 Dec 2023 11:30:20 +0100 Subject: [PATCH 6/7] use subclass + env_var --- conda_libmamba_solver/index.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index f0c8b756..c0756373 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -81,8 +81,8 @@ import libmambapy as api from conda import __version__ as conda_version from conda.base.constants import REPODATA_FN -from conda.base.context import context -from conda.common.io import DummyExecutor, ThreadLimitedThreadPoolExecutor +from conda.base.context import context, reset_context +from conda.common.io import DummyExecutor, ThreadLimitedThreadPoolExecutor, env_var from conda.common.serialize import json_dump, json_load from conda.common.url import percent_decode, remove_auth, split_anaconda_token from conda.core.subdir_data import SubdirData @@ -409,11 +409,8 @@ def _process_query_result( return result -# for conda-build -_CachedLibMambaIndexHelper = lru_cache(maxsize=None)(LibMambaIndexHelper) - - -def _LibMambaIndexForCondaBuild(*args, **kwargs): +@lru_cache(maxsize=None) +class _LibMambaIndexForCondaBuild(LibMambaIndexHelper): """ See https://github.com/conda/conda-libmamba-solver/issues/386 @@ -423,14 +420,15 @@ def _LibMambaIndexForCondaBuild(*args, **kwargs): the condarc configuration might be ignored, resulting in bad index configuration and missing packages anyway. """ - if VersionOrder(conda_version) <= VersionOrder("23.10.0"): - log.warning( - "conda-build requires conda >=23.11.0 for offline index support. " - "Falling back to online index. This might result in KeyError messages, " - "specially if the remote repodata is updated during the build phase. " - "See https://github.com/conda/conda-libmamba-solver/issues/386." - ) - return _CachedLibMambaIndexHelper(*args, **kwargs) - - with context._override("offline", True): - return _CachedLibMambaIndexHelper(*args, **kwargs) + def __init__(self, *args, **kwargs): + if VersionOrder(conda_version) <= VersionOrder("23.10.0"): + log.warning( + "conda-build requires conda >=23.11.0 for offline index support. " + "Falling back to online index. This might result in KeyError messages, " + "specially if the remote repodata is updated during the build phase. " + "See https://github.com/conda/conda-libmamba-solver/issues/386." + ) + super().__init__(*args, **kwargs) + else: + with env_var("CONDA_OFFLINE", "true", callback=reset_context): + super().__init__(*args, **kwargs) From bead9fa195c264de338f1cadfbc0fc23b1859ec3 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 5 Dec 2023 11:51:55 +0100 Subject: [PATCH 7/7] pre-commit --- conda_libmamba_solver/index.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index c0756373..b8bfe1c2 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -420,6 +420,7 @@ class _LibMambaIndexForCondaBuild(LibMambaIndexHelper): the condarc configuration might be ignored, resulting in bad index configuration and missing packages anyway. """ + def __init__(self, *args, **kwargs): if VersionOrder(conda_version) <= VersionOrder("23.10.0"): log.warning(