From a246813bf5c7e35d03458bc4c04c0c32ae13db77 Mon Sep 17 00:00:00 2001 From: Dan Blanchard Date: Fri, 5 Jan 2024 14:54:36 -0500 Subject: [PATCH 01/10] Add support for egg packages with files outside site-packages --- importlib_metadata/__init__.py | 24 ++++++++++++++++++++---- importlib_metadata/_meta.py | 3 +++ tests/fixtures.py | 34 ++++++++++++++++++++++++++++++++++ tests/test_api.py | 1 + 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index bbd92a7d..42230385 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -562,15 +562,31 @@ def _read_files_egginfo_installed(self): if not text or not subdir: return + site_path = self.locate_file('').resolve() paths = ( - (subdir / name) - .resolve() - .relative_to(self.locate_file('').resolve()) - .as_posix() + self._relative_to( + (subdir / name).resolve(), + site_path, + ).as_posix() for name in text.splitlines() ) return map('"{}"'.format, paths) + def _relative_to(self, path, root): + """ + Workaround for https://bugs.python.org/issue23082 where ".." + isn't added by pathlib.Path.relative_to() when path is not + a subpath of root. + + One example of such a package is dask-labextension, which uses + jupyter-packaging to install JupyterLab javascript files outside + of site-packages. + """ + try: + return path.relative_to(root) + except ValueError: + return pathlib.Path(os.path.relpath(path, root)) + def _read_files_egginfo_sources(self): """ Read SOURCES.txt and return lines in a similar CSV-parsable diff --git a/importlib_metadata/_meta.py b/importlib_metadata/_meta.py index 1342d839..e3892da4 100644 --- a/importlib_metadata/_meta.py +++ b/importlib_metadata/_meta.py @@ -70,3 +70,6 @@ def read_bytes(self) -> bytes: def exists(self) -> bool: ... # pragma: no cover + + def resolve(self) -> bool: + ... # pragma: no cover diff --git a/tests/fixtures.py b/tests/fixtures.py index b419d81c..b194864a 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -252,6 +252,40 @@ def main(): } +class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module.json + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../../../etc/jupyter/jupyter_notebook_config.d/egg_with_module.json + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } + + + class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): files: FilesSpec = { "egg_with_no_modules_pkg.egg-info": { diff --git a/tests/test_api.py b/tests/test_api.py index a85c62ad..1280c532 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -29,6 +29,7 @@ class APITests( fixtures.EggInfoPkg, fixtures.EggInfoPkgPipInstalledNoToplevel, fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgPipInstalledExternalDataFiles, fixtures.EggInfoPkgSourcesFallback, fixtures.DistInfoPkg, fixtures.DistInfoPkgWithDot, From d3a1333ce57379af65b75a13b7a4981e1d4d0c42 Mon Sep 17 00:00:00 2001 From: Dan Blanchard Date: Wed, 10 Jan 2024 10:35:13 -0500 Subject: [PATCH 02/10] Tweak test --- tests/fixtures.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index b194864a..263d5ffe 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -270,7 +270,8 @@ class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): # accurate source than SOURCES.txt as to the installed contents of # the package. "installed-files.txt": """ - ../../../etc/jupyter/jupyter_notebook_config.d/egg_with_module.json + ../../../etc/jupyter/jupyter_notebook_config.d/relative.json + /etc/jupyter/jupyter_notebook_config.d/absolute.json ../egg_with_module.py PKG-INFO SOURCES.txt From 608b319e33e170f746b32061de5b8862e0a943f7 Mon Sep 17 00:00:00 2001 From: Dan Blanchard Date: Thu, 7 Mar 2024 09:28:51 -0500 Subject: [PATCH 03/10] Update importlib_metadata/__init__.py Co-authored-by: Jason R. Coombs --- importlib_metadata/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 4bd9093c..dc23148f 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -574,7 +574,7 @@ def _read_files_egginfo_installed(self): def _relative_to(self, path, root): """ - Workaround for https://bugs.python.org/issue23082 where ".." + Workaround for https://github.com/python/cpython/issues/67271 where ".." isn't added by pathlib.Path.relative_to() when path is not a subpath of root. From 4a3782d11895c22fd443c67175ab7352db2a769b Mon Sep 17 00:00:00 2001 From: Dan Blanchard Date: Thu, 7 Mar 2024 10:22:30 -0500 Subject: [PATCH 04/10] Fix mypy failure that has nothing to do with this PR --- importlib_metadata/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index dc23148f..61325221 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -440,7 +440,7 @@ def at(path: str | os.PathLike[str]) -> Distribution: :param path: a string or path-like object :return: a concrete Distribution instance for the path """ - return PathDistribution(pathlib.Path(path)) + return PathDistribution(cast(SimplePath, pathlib.Path(path))) @staticmethod def _discover_resolvers(): From 08d74cc165094bda158e108747fa39ae10674bbf Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 22 Jun 2024 21:42:03 -0400 Subject: [PATCH 05/10] Move _relative_to compatibility to a compat module. --- importlib_metadata/__init__.py | 24 ++++-------------------- importlib_metadata/compat/py38.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 importlib_metadata/compat/py38.py diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 61325221..c2c4b1e9 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -19,6 +19,7 @@ import posixpath import collections +from .compat.py38 import relative_fix from . import _adapters, _meta, _py39compat from ._collections import FreezableDefaultDict, Pair from ._compat import ( @@ -562,31 +563,14 @@ def _read_files_egginfo_installed(self): if not text or not subdir: return - site_path = self.locate_file('').resolve() paths = ( - self._relative_to( - (subdir / name).resolve(), - site_path, - ).as_posix() + relative_fix((subdir / name).resolve()) + .relative_to(self.locate_file('').resolve(), walk_up=True) + .as_posix() for name in text.splitlines() ) return map('"{}"'.format, paths) - def _relative_to(self, path, root): - """ - Workaround for https://github.com/python/cpython/issues/67271 where ".." - isn't added by pathlib.Path.relative_to() when path is not - a subpath of root. - - One example of such a package is dask-labextension, which uses - jupyter-packaging to install JupyterLab javascript files outside - of site-packages. - """ - try: - return path.relative_to(root) - except ValueError: - return pathlib.Path(os.path.relpath(path, root)) - def _read_files_egginfo_sources(self): """ Read SOURCES.txt and return lines in a similar CSV-parsable diff --git a/importlib_metadata/compat/py38.py b/importlib_metadata/compat/py38.py new file mode 100644 index 00000000..2499c3f6 --- /dev/null +++ b/importlib_metadata/compat/py38.py @@ -0,0 +1,23 @@ +import os +import pathlib +import sys +import types + + +def wrap(path): + """ + Workaround for https://github.com/python/cpython/issues/67271 where ".." + isn't added by pathlib.Path.relative_to() when path is not + a subpath of root. + One example of such a package is dask-labextension, which uses + jupyter-packaging to install JupyterLab javascript files outside + of site-packages. + """ + + def relative_to(root, *, walk_up=False): + return pathlib.Path(os.path.relpath(path, root)) + + return types.SimpleNamespace(relative_to=relative_to) + + +relative_fix = wrap if sys.version_info < (3, 9) else lambda x: x From 1584b96f50317f7cbf55192706a9a1c4ef869a3b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 22 Jun 2024 21:56:20 -0400 Subject: [PATCH 06/10] Update wrapper to rely on 'relative_to(walk_up=True)' on Python 3.12 and compatibility wrapper on Python 3.11 and earlier. --- importlib_metadata/__init__.py | 2 +- importlib_metadata/compat/{py38.py => py311.py} | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) rename importlib_metadata/compat/{py38.py => py311.py} (53%) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index c2c4b1e9..94818f62 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -19,7 +19,7 @@ import posixpath import collections -from .compat.py38 import relative_fix +from .compat.py311 import relative_fix from . import _adapters, _meta, _py39compat from ._collections import FreezableDefaultDict, Pair from ._compat import ( diff --git a/importlib_metadata/compat/py38.py b/importlib_metadata/compat/py311.py similarity index 53% rename from importlib_metadata/compat/py38.py rename to importlib_metadata/compat/py311.py index 2499c3f6..d4b6f359 100644 --- a/importlib_metadata/compat/py38.py +++ b/importlib_metadata/compat/py311.py @@ -6,10 +6,9 @@ def wrap(path): """ - Workaround for https://github.com/python/cpython/issues/67271 where ".." - isn't added by pathlib.Path.relative_to() when path is not - a subpath of root. - One example of such a package is dask-labextension, which uses + Workaround for https://github.com/python/cpython/issues/84538 + to add backward compatibility for walk_up=True. + An example affected package is dask-labextension, which uses jupyter-packaging to install JupyterLab javascript files outside of site-packages. """ @@ -20,4 +19,4 @@ def relative_to(root, *, walk_up=False): return types.SimpleNamespace(relative_to=relative_to) -relative_fix = wrap if sys.version_info < (3, 9) else lambda x: x +relative_fix = wrap if sys.version_info < (3, 12) else lambda x: x From b815aee5352ed728f6f90ba7362f3dddf46ab418 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 22 Jun 2024 22:18:37 -0400 Subject: [PATCH 07/10] Mark compat code as uncovered. --- importlib_metadata/compat/py311.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/importlib_metadata/compat/py311.py b/importlib_metadata/compat/py311.py index d4b6f359..3a532743 100644 --- a/importlib_metadata/compat/py311.py +++ b/importlib_metadata/compat/py311.py @@ -4,7 +4,7 @@ import types -def wrap(path): +def wrap(path): # pragma: no cover """ Workaround for https://github.com/python/cpython/issues/84538 to add backward compatibility for walk_up=True. From 07a2a4402fb39f03facea611fa9da8d9b927602e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 23 Jun 2024 10:34:14 -0400 Subject: [PATCH 08/10] Revert "Fix mypy failure that has nothing to do with this PR" This reverts commit 4a3782d11895c22fd443c67175ab7352db2a769b. --- importlib_metadata/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index f79376e9..eab73ffe 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -440,7 +440,7 @@ def at(path: str | os.PathLike[str]) -> Distribution: :param path: a string or path-like object :return: a concrete Distribution instance for the path """ - return PathDistribution(cast(SimplePath, pathlib.Path(path))) + return PathDistribution(pathlib.Path(path)) @staticmethod def _discover_resolvers(): From e4d1dcca7244c0d890c57eb24b3b8a6a76f4910e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 23 Jun 2024 11:02:40 -0400 Subject: [PATCH 09/10] Remove additional method in SimplePath. --- importlib_metadata/_meta.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/importlib_metadata/_meta.py b/importlib_metadata/_meta.py index 13ab8d90..1927d0f6 100644 --- a/importlib_metadata/_meta.py +++ b/importlib_metadata/_meta.py @@ -65,5 +65,3 @@ def read_text(self, encoding=None) -> str: ... # pragma: no cover def read_bytes(self) -> bytes: ... # pragma: no cover def exists(self) -> bool: ... # pragma: no cover - - def resolve(self) -> bool: ... # pragma: no cover From b94b42ef3103250a0f509f68170037199dc86583 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 23 Jun 2024 11:10:58 -0400 Subject: [PATCH 10/10] Add news fragment --- newsfragments/455.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/455.bugfix.rst diff --git a/newsfragments/455.bugfix.rst b/newsfragments/455.bugfix.rst new file mode 100644 index 00000000..72bc9917 --- /dev/null +++ b/newsfragments/455.bugfix.rst @@ -0,0 +1 @@ +When reading installed files from an egg, use ``relative_to(walk_up=True)`` to honor files installed outside of the installation root. \ No newline at end of file