From 331a6554889134c253ed22141562f4f0b4d0e288 Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Mon, 1 Jul 2024 11:46:19 +0000 Subject: [PATCH 01/12] Add memray to ci --- .github/workflows/tests.yml | 71 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 72 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 78ba210f92..a7a2dcf2eb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -234,6 +234,77 @@ jobs: token: ${{ secrets.CODSPEED_TOKEN }} run: $CONDA/envs/test/bin/pytest --codspeed + # linux memray + linux-memray: + # only run test suite if there are code changes + needs: changes + if: needs.changes.outputs.code == 'true' + + runs-on: ubuntu-latest + defaults: + run: + # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell + shell: bash -el {0} # bash exit immediately on error + login shell + strategy: + fail-fast: false + matrix: + python-version: ['3.12'] + + steps: + - name: Checkout Source + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Hash + Timestamp + run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-memray-$(date -u "+%Y%m")" >> $GITHUB_ENV + + - name: Cache Conda + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 + with: + path: ~/conda_pkgs_dir + key: cache-${{ env.HASH }} + + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca + with: + condarc-file: .github/condarc + run-post: false # skip post cleanup + + - name: Conda Install + run: > + conda install + --yes + --file tests/requirements.txt + --file tests/requirements-${{ runner.os }}.txt + --file tests/requirements-ci.txt + python=${{ matrix.python-version }} + ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }} + pytest-memray + + - name: Install CodSpeed + run: pip install git+https://github.com/kenodegard/pytest-codspeed.git@fix-outerr-redirects#egg=pytest-codspeed + + - name: Install development conda + run: pip install git+https://github.com/zklaus/conda@lazy-index && python -m conda init --install + + # TODO: how can we remove this step? + - name: Install Self + run: pip install -e . + + - name: Conda Info + # view test env info (not base) + run: python -m conda info --verbose + + - name: Conda Config + run: conda config --show-sources + + - name: Conda List + run: conda list --show-channel-urls + + - name: Run memray + run: python -m pytest -k memray --memray + # windows test suite windows: # only run test suite if there are code changes diff --git a/pyproject.toml b/pyproject.toml index 12fe4731f6..e9ea40b6f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,7 @@ markers = [ "sanity: execute the sanity tests", "no_default_testing_config: used internally to disable monkeypatching for testing_config", "benchmark: execute the benchmark tests", + "memray: memory use tests", ] minversion = 3.0 norecursedirs = ["tests/test-recipes/*"] From 1819c4c6652ce1983a0a87d79c14fc684694f929 Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Wed, 5 Jun 2024 08:44:32 +0000 Subject: [PATCH 02/12] Replace use of get_index w/ Index --- conda_build/index.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/conda_build/index.py b/conda_build/index.py index bcb9c6a9d0..9edc40ce7b 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -6,7 +6,7 @@ from os.path import dirname from conda.base.context import context -from conda.core.index import get_index +from conda.core.index import Index from conda.exceptions import CondaHTTPError from conda.utils import url_path from conda_index.index import update_index as _update_index @@ -102,26 +102,26 @@ def get_build_index( if subdir == "noarch": subdir = context.subdir try: - # get_index() is like conda reading the index, not conda_index + # Index() is like conda reading the index, not conda_index # creating a new index. - cached_index = get_index( - channel_urls=urls, + cached_index = Index( + channels=urls, prepend=not omit_defaults, + platform=subdir, use_local=False, use_cache=context.offline, - platform=subdir, ) # HACK: defaults does not have the many subfolders we support. Omit it and # try again. except CondaHTTPError: if "defaults" in urls: urls.remove("defaults") - cached_index = get_index( - channel_urls=urls, - prepend=omit_defaults, + cached_index = Index( + channels=urls, + prepend=not omit_defaults, + platform=subdir, use_local=False, use_cache=context.offline, - platform=subdir, ) local_index_timestamp = os.path.getmtime(index_file) From 057daa6f4809f269a958ee7eb9a29cb64cbb6478 Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Wed, 5 Jun 2024 14:57:28 +0000 Subject: [PATCH 03/12] Target conda dev branch in tests action --- .github/workflows/tests.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a7a2dcf2eb..2879a7fed3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -131,6 +131,19 @@ jobs: python=${{ matrix.python-version }} ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }} + - name: Checkout development conda + uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 + with: + repository: zklaus/conda.git + ref: lazy-index + path: conda-src + + - name: Install development conda + run: | + cd conda-src + pip install -e . + cd .. + # TODO: how can we remove this step? - name: Install Self run: pip install -e . From 3b8ea4eaeb6f90511f7ac10e35a20789a2f4ee98 Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Wed, 5 Jun 2024 15:00:01 +0000 Subject: [PATCH 04/12] Make sure to get tags --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2879a7fed3..36d510f5cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -137,6 +137,7 @@ jobs: repository: zklaus/conda.git ref: lazy-index path: conda-src + fetch-depth: 0 - name: Install development conda run: | From 2f9f913642d13a897d9beeaddbd48c2b80583ede Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Tue, 18 Jun 2024 14:25:49 +0000 Subject: [PATCH 05/12] Simplify CI --- .github/workflows/tests.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36d510f5cb..b8b529da8f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -131,18 +131,10 @@ jobs: python=${{ matrix.python-version }} ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }} - - name: Checkout development conda - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 - with: - repository: zklaus/conda.git - ref: lazy-index - path: conda-src - fetch-depth: 0 - - name: Install development conda run: | cd conda-src - pip install -e . + pip install git+https://github.com/zklaus/conda@lazy-index && python -m conda init --install cd .. # TODO: how can we remove this step? From 87330395488f1fcf6e405ef799732686d17e5fb9 Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Tue, 18 Jun 2024 14:28:27 +0000 Subject: [PATCH 06/12] Fix CI --- .github/workflows/tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b8b529da8f..dd8604b58a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -132,10 +132,7 @@ jobs: ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }} - name: Install development conda - run: | - cd conda-src - pip install git+https://github.com/zklaus/conda@lazy-index && python -m conda init --install - cd .. + run: pip install git+https://github.com/zklaus/conda@lazy-index && python -m conda init --install # TODO: how can we remove this step? - name: Install Self From deacf0b264adcb62b621fe5d45558352c9dd3efa Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Wed, 19 Jun 2024 11:19:32 +0000 Subject: [PATCH 07/12] Replacing remaining uses of get_index --- conda_build/inspect_pkg.py | 4 ++-- conda_build/skeletons/cpan.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index 54ec2c6f28..2f2ec97e0a 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -16,7 +16,7 @@ from conda.api import Solver from conda.base.context import context from conda.cli.common import specs_from_args -from conda.core.index import get_index +from conda.core.index import Index from conda.core.prefix_data import PrefixData from conda.models.records import PrefixRecord @@ -173,7 +173,7 @@ def test_installable(channel: str = "defaults") -> bool: success = True for subdir in ["osx-64", "linux-32", "linux-64", "win-32", "win-64"]: log.info("######## Testing subdir %s ########", subdir) - for prec in get_index(channel_urls=[channel], prepend=False, platform=subdir): + for prec in Index(channels=[channel], prepend=False, platform=subdir): name = prec["name"] if name in {"conda", "conda-build"}: # conda can only be installed in the root environment diff --git a/conda_build/skeletons/cpan.py b/conda_build/skeletons/cpan.py index 31213054d1..0ac5395f65 100644 --- a/conda_build/skeletons/cpan.py +++ b/conda_build/skeletons/cpan.py @@ -21,7 +21,7 @@ from os.path import basename, dirname, exists, join import requests -from conda.core.index import get_index +from conda.core.index import Index from conda.exceptions import CondaError, CondaHTTPError from conda.gateways.connection.download import TmpDownload, download from conda.gateways.disk.create import TemporaryDirectory @@ -695,7 +695,7 @@ def latest_pkg_version(pkg): """ :returns: the latest version of the specified conda package available """ - r = Resolve(get_index()) + r = Resolve(Index()) try: pkg_list = sorted(r.get_pkgs(MatchSpec(pkg))) except: From b3e9f99d3853751a2dcbcb42d9082a558a989d82 Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Wed, 19 Jun 2024 11:43:39 +0000 Subject: [PATCH 08/12] Filter python 3.12 tarfile deprecation warning --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index e9ea40b6f0..fbae645f0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,6 +124,7 @@ filterwarnings = [ # elevate conda-build's deprecated warning to an error "error::PendingDeprecationWarning:conda_build", "error::DeprecationWarning:conda_build", + "ignore:Python 3.14 will, by default, filter extracted tar archives:DeprecationWarning:conda_package_streaming", # ignore numpy.distutils error 'ignore:\s+`numpy.distutils` is deprecated:DeprecationWarning:conda_build._load_setup_py_data', # ignore conda-index error From 3faaf1d22c1a6f808fc9f3a5f2688ed930bf7ba8 Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Tue, 25 Jun 2024 13:27:03 +0000 Subject: [PATCH 09/12] Adapt test to new logging --- tests/test_api_build.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index efba89d75d..72442f277e 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -1826,7 +1826,7 @@ def test_downstream_tests(testing_config): @pytest.mark.sanity -def test_warning_on_file_clobbering(testing_config, capfd): +def test_warning_on_file_clobbering(testing_config, caplog): recipe_dir = os.path.join(metadata_dir, "_overlapping_files_warning") api.build( @@ -1844,8 +1844,11 @@ def test_warning_on_file_clobbering(testing_config, capfd): config=testing_config, ) # The clobber warning here is raised when creating the test environment for b - out, err = capfd.readouterr() - assert "ClobberWarning" in err + clobber_warning_found = False + for record in caplog.records: + if "ClobberWarning:" in record.message: + clobber_warning_found = True + assert clobber_warning_found with pytest.raises((ClobberError, CondaMultiError)): with env_var("CONDA_PATH_CONFLICT", "prevent", reset_context): api.build(os.path.join(recipe_dir, "b"), config=testing_config) From cd97efef04bc16f4bd51a85dda296b6d64f43655 Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Tue, 25 Jun 2024 13:30:53 +0000 Subject: [PATCH 10/12] Use development conda on all ci platforms --- .github/workflows/tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dd8604b58a..e718e1fc11 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -217,6 +217,9 @@ jobs: - name: Install CodSpeed run: pip install git+https://github.com/kenodegard/pytest-codspeed.git@fix-outerr-redirects#egg=pytest-codspeed + - name: Install development conda + run: pip install git+https://github.com/zklaus/conda@lazy-index && python -m conda init --install + # TODO: how can we remove this step? - name: Install Self run: pip install -e . @@ -369,6 +372,9 @@ jobs: python=${{ matrix.python-version }} ${{ env.CONDA_CHANNEL_LABEL }}::conda + - name: Install development conda + run: pip install git+https://github.com/zklaus/conda@lazy-index && python -m conda init --install + # TODO: how can we remove this step? - name: Install Self run: pip install -e . @@ -488,6 +494,9 @@ jobs: python=${{ matrix.python-version }} ${{ env.CONDA_CHANNEL_LABEL }}::conda + - name: Install development conda + run: pip install git+https://github.com/zklaus/conda@lazy-index && python -m conda init --install + # TODO: how can we remove this step? - name: Install Self run: pip install -e . From 02bed68fe85d8daad73a2fa93a11f64d5264533f Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Fri, 26 Jul 2024 12:49:52 +0000 Subject: [PATCH 11/12] Remove unused use_cache parameter --- conda_build/index.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/conda_build/index.py b/conda_build/index.py index 9edc40ce7b..04e20b47e4 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -109,7 +109,6 @@ def get_build_index( prepend=not omit_defaults, platform=subdir, use_local=False, - use_cache=context.offline, ) # HACK: defaults does not have the many subfolders we support. Omit it and # try again. @@ -121,7 +120,6 @@ def get_build_index( prepend=not omit_defaults, platform=subdir, use_local=False, - use_cache=context.offline, ) local_index_timestamp = os.path.getmtime(index_file) From 739c60a52fa3c1b590a90ff8eea678709b572cfb Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Mon, 29 Jul 2024 10:20:06 +0000 Subject: [PATCH 12/12] Ignore two pendingdeprecation warnings --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fbae645f0f..da1ef47e8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,6 +131,8 @@ filterwarnings = [ "ignore::PendingDeprecationWarning:conda_index", "ignore::DeprecationWarning:conda_index", "ignore:Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata:DeprecationWarning", + "ignore:conda.core.index._supplement_index_with_system is pending deprecation:PendingDeprecationWarning:conda", + "ignore:conda.core.index._make_virtual_package is pending deprecation:PendingDeprecationWarning:conda", ] markers = [ "serial: execute test serially (to avoid race conditions)",