From 6c8496a25889f60de338d009699cb6ff345fcdea Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 5 Jun 2024 13:21:28 -0500 Subject: [PATCH 1/8] filter platform out of filename when finding .conda component files --- conda_package_streaming/package_streaming.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conda_package_streaming/package_streaming.py b/conda_package_streaming/package_streaming.py index e0c5b78..525d6e5 100644 --- a/conda_package_streaming/package_streaming.py +++ b/conda_package_streaming/package_streaming.py @@ -7,6 +7,7 @@ import bz2 import os import os.path +import re import tarfile import zipfile from enum import Enum @@ -125,9 +126,12 @@ def stream_conda_component( zf = zipfile.ZipFile(fileobj or filename) file_id, _, _ = os.path.basename(filename).rpartition(".") + # this substitution compensates for web downloads from anaconda.org having + # the platform as a prefix + file_id = re.sub("^(osx|linux|win)-.+?_", "", file_id) component_name = f"{component}-{file_id}" component_filename = [ - info for info in zf.infolist() if info.filename.startswith(component_name) + info for info in zf.infolist() if component_name in info.filename ] if not component_filename: raise LookupError(f"didn't find {component_name} component in {filename}") From 5a956ad72dc17a8d9d497941a4927bb918ca5b66 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 5 Jun 2024 13:47:09 -0500 Subject: [PATCH 2/8] adapt for noarch also --- conda_package_streaming/package_streaming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_package_streaming/package_streaming.py b/conda_package_streaming/package_streaming.py index 525d6e5..7125bec 100644 --- a/conda_package_streaming/package_streaming.py +++ b/conda_package_streaming/package_streaming.py @@ -128,7 +128,7 @@ def stream_conda_component( file_id, _, _ = os.path.basename(filename).rpartition(".") # this substitution compensates for web downloads from anaconda.org having # the platform as a prefix - file_id = re.sub("^(osx|linux|win)-.+?_", "", file_id) + file_id = re.sub("^(osx|linux|win|noarch)(-.+?)?_", "", file_id) component_name = f"{component}-{file_id}" component_filename = [ info for info in zf.infolist() if component_name in info.filename From 1cf10cd38b2de6a8b284e13eec68a2cd7d169599 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 5 Jun 2024 13:54:14 -0500 Subject: [PATCH 3/8] revert filename 'in' vs 'startswith' change --- conda_package_streaming/package_streaming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_package_streaming/package_streaming.py b/conda_package_streaming/package_streaming.py index 7125bec..49557b8 100644 --- a/conda_package_streaming/package_streaming.py +++ b/conda_package_streaming/package_streaming.py @@ -131,7 +131,7 @@ def stream_conda_component( file_id = re.sub("^(osx|linux|win|noarch)(-.+?)?_", "", file_id) component_name = f"{component}-{file_id}" component_filename = [ - info for info in zf.infolist() if component_name in info.filename + info for info in zf.infolist() if info.filename.startswith(component_name) ] if not component_filename: raise LookupError(f"didn't find {component_name} component in {filename}") From 2427ff8a51efc9aa4867570ff558476a76f5d354 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 5 Jun 2024 14:06:30 -0500 Subject: [PATCH 4/8] add tests using renamed cached packages --- tests/test_streaming.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_streaming.py b/tests/test_streaming.py index e6877b0..37d07f0 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -1,5 +1,7 @@ import io import json +import os +import shutil import tarfile import pytest @@ -7,6 +9,16 @@ from conda_package_streaming import package_streaming +@pytest.mark.parametrize("subdir", ["linux-64", "noarch", "win-32", "osx-arm64"]) +def test_package_streaming_with_subdir_prefix(conda_paths, tmp_path, subdir): + """Regression test for https://github.com/conda/conda-package-handling/issues/230""" + for path in conda_paths: + copied_file = tmp_path / f"{subdir}_{os.path.basename(path)}" + shutil.copyfile(path, copied_file ) + + if str(path).endswith(".conda"): + package_streaming.stream_conda_component(copied_file, component="info") + def test_package_streaming(conda_paths): for path in conda_paths: if str(path).endswith(".conda"): From 7362a5cd65487698965c0e0c3871356a4ff1fe0d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 5 Jun 2024 14:09:22 -0500 Subject: [PATCH 5/8] black --- tests/test_streaming.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_streaming.py b/tests/test_streaming.py index 37d07f0..736177b 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -14,11 +14,12 @@ def test_package_streaming_with_subdir_prefix(conda_paths, tmp_path, subdir): """Regression test for https://github.com/conda/conda-package-handling/issues/230""" for path in conda_paths: copied_file = tmp_path / f"{subdir}_{os.path.basename(path)}" - shutil.copyfile(path, copied_file ) + shutil.copyfile(path, copied_file) if str(path).endswith(".conda"): package_streaming.stream_conda_component(copied_file, component="info") + def test_package_streaming(conda_paths): for path in conda_paths: if str(path).endswith(".conda"): From b6db31be82c0221267000c064a8f188997989321 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Wed, 5 Jun 2024 16:07:23 -0400 Subject: [PATCH 6/8] prefix/suffix strategy without platform lists --- conda_package_streaming/package_streaming.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/conda_package_streaming/package_streaming.py b/conda_package_streaming/package_streaming.py index 49557b8..aa138e5 100644 --- a/conda_package_streaming/package_streaming.py +++ b/conda_package_streaming/package_streaming.py @@ -126,15 +126,19 @@ def stream_conda_component( zf = zipfile.ZipFile(fileobj or filename) file_id, _, _ = os.path.basename(filename).rpartition(".") - # this substitution compensates for web downloads from anaconda.org having - # the platform as a prefix - file_id = re.sub("^(osx|linux|win|noarch)(-.+?)?_", "", file_id) - component_name = f"{component}-{file_id}" + + # deal with file_id possibly having a _ prefix... compensates for + # web downloads from anaconda.org having the platform as a prefix + component_prefix = f"{component}-" + component_suffix = f"{file_id}.tar.zst" component_filename = [ - info for info in zf.infolist() if info.filename.startswith(component_name) + info + for info in zf.infolist() + if info.filename.startswith(component_prefix) + and component_suffix.endswith(info.filename.split("-", 1)[-1]) ] if not component_filename: - raise LookupError(f"didn't find {component_name} component in {filename}") + raise LookupError(f"didn't find {component} component in {filename}") assert len(component_filename) == 1 reader = zstandard.ZstdDecompressor().stream_reader( zf.open(component_filename[0]) From 2de16adf3f72b25b11bdfc140be21cdc6d4a4292 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Wed, 5 Jun 2024 16:08:01 -0400 Subject: [PATCH 7/8] remove unused import --- conda_package_streaming/package_streaming.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_package_streaming/package_streaming.py b/conda_package_streaming/package_streaming.py index aa138e5..7e1a429 100644 --- a/conda_package_streaming/package_streaming.py +++ b/conda_package_streaming/package_streaming.py @@ -7,7 +7,6 @@ import bz2 import os import os.path -import re import tarfile import zipfile from enum import Enum From 0d2ce269d1b241a338fba1935a2d879d5fbe83ed Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Wed, 5 Jun 2024 16:24:04 -0400 Subject: [PATCH 8/8] update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e0db86..2f0525e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ [//]: # (current developments) +## 0.11.0 + +* Relax `package_streaming.stream_conda_component` to find inner component + archives even if a prefix `name-1.0-a_0.conda` has been added to the + `.conda` filename. + (https://github.com/conda/conda-package-handling/issues/230) + ## 0.10.0 (2024-06) * Use zip64 extensions when converting .tar.bz2 to .conda, if uncompressed size