diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index e38c5aa9e7..cbb60d4f25 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -24,7 +24,7 @@ get_package_obj_files, get_untracked_obj_files, ) -from conda_build.os_utils.liefldd import codefile_type +from conda_build.os_utils.liefldd import codefile_class, machofile from conda_build.os_utils.macho import get_rpaths, human_filetype from conda_build.utils import ( comma_join, @@ -354,14 +354,16 @@ def inspect_objects(packages, prefix=sys.prefix, groupby="package"): info = [] for f in obj_files: - f_info = {} path = join(prefix, f) - filetype = codefile_type(path) - if filetype == "machofile": - f_info["filetype"] = human_filetype(path, None) - f_info["rpath"] = ":".join(get_rpaths(path)) - f_info["filename"] = f - info.append(f_info) + codefile = codefile_class(path) + if codefile == machofile: + info.append( + { + "filetype": human_filetype(path, None), + "rpath": ":".join(get_rpaths(path)), + "filename": f, + } + ) output_string += print_object_info(info, groupby) if hasattr(output_string, "decode"): diff --git a/conda_build/os_utils/ldd.py b/conda_build/os_utils/ldd.py index 77daf4ab10..32eea125a2 100644 --- a/conda_build/os_utils/ldd.py +++ b/conda_build/os_utils/ldd.py @@ -8,12 +8,7 @@ from conda_build.conda_interface import linked_data, untracked from conda_build.os_utils.macho import otool -from conda_build.os_utils.pyldd import ( - codefile_class, - inspect_linkages, - is_codefile, - machofile, -) +from conda_build.os_utils.pyldd import codefile_class, inspect_linkages, machofile LDD_RE = re.compile(r"\s*(.*?)\s*=>\s*(.*?)\s*\(.*\)") LDD_NOT_FOUND_RE = re.compile(r"\s*(.*?)\s*=>\s*not found") @@ -118,7 +113,7 @@ def get_package_obj_files(dist, prefix): files = get_package_files(dist, prefix) for f in files: path = join(prefix, f) - if is_codefile(path): + if codefile_class(path): res.append(f) return res @@ -130,7 +125,7 @@ def get_untracked_obj_files(prefix): files = untracked(prefix) for f in files: path = join(prefix, f) - if is_codefile(path): + if codefile_class(path): res.append(f) return res diff --git a/conda_build/os_utils/liefldd.py b/conda_build/os_utils/liefldd.py index 2cf6ce92ad..26a768a4f6 100644 --- a/conda_build/os_utils/liefldd.py +++ b/conda_build/os_utils/liefldd.py @@ -1,9 +1,6 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause -try: - from collections.abc import Hashable -except ImportError: - from collections.abc import Hashable +from __future__ import annotations import hashlib import json @@ -11,34 +8,34 @@ import struct import sys import threading +from collections.abc import Hashable from fnmatch import fnmatch from functools import partial +from pathlib import Path from subprocess import PIPE, Popen +from ..deprecations import deprecated from .external import find_executable # lief cannot handle files it doesn't know about gracefully # TODO :: Remove all use of pyldd # Currently we verify the output of each against the other -from .pyldd import codefile_type as codefile_type_pyldd +from .pyldd import DLLfile, EXEfile, elffile, machofile +from .pyldd import codefile_type as _codefile_type from .pyldd import inspect_linkages as inspect_linkages_pyldd -codefile_type = codefile_type_pyldd -have_lief = False try: import lief lief.logging.disable() have_lief = True -except: - pass +except ImportError: + have_lief = False +@deprecated("3.28.0", "4.0.0", addendum="Use `isinstance(value, str)` instead.") def is_string(s): - try: - return isinstance(s, basestring) - except NameError: - return isinstance(s, str) + return isinstance(s, str) # Some functions can operate on either file names @@ -46,17 +43,16 @@ def is_string(s): # these are to be avoided, or if not avoided they # should be passed a binary when possible as that # will prevent having to parse it multiple times. -def ensure_binary(file): - if not is_string(file): +def ensure_binary(file: str | os.PathLike | Path | lief.Binary) -> lief.Binary | None: + if isinstance(file, lief.Binary): return file - else: - try: - if not os.path.exists(file): - return [] - return lief.parse(file) - except: - print(f"WARNING: liefldd: failed to ensure_binary({file})") - return None + elif not Path(file).exists(): + return None + try: + return lief.parse(str(file)) + except BaseException: + print(f"WARNING: liefldd: failed to ensure_binary({file})") + return None def nm(filename): @@ -77,25 +73,57 @@ def nm(filename): print("No symbols found") -def codefile_type_liefldd(file, skip_symlinks=True): - binary = ensure_binary(file) - result = None - if binary: - if binary.format == lief.EXE_FORMATS.PE: - if lief.PE.DLL_CHARACTERISTICS: - if binary.header.characteristics & lief.PE.HEADER_CHARACTERISTICS.DLL: - result = "DLLfile" - else: - result = "EXEfile" +if have_lief: + + def codefile_class( + path: str | os.PathLike | Path, + skip_symlinks: bool = False, + ) -> type[DLLfile | EXEfile | machofile | elffile] | None: + # same signature as conda.os_utils.pyldd.codefile_class + if not (binary := ensure_binary(path)): + return None + elif ( + binary.format == lief.EXE_FORMATS.PE + and lief.PE.HEADER_CHARACTERISTICS.DLL in binary.header.characteristics_list + ): + return DLLfile + elif binary.format == lief.EXE_FORMATS.PE: + return EXEfile elif binary.format == lief.EXE_FORMATS.MACHO: - result = "machofile" + return machofile elif binary.format == lief.EXE_FORMATS.ELF: - result = "elffile" - return result - + return elffile + else: + return None -if have_lief: - codefile_type = codefile_type_liefldd +else: + from .pyldd import codefile_class + + +@deprecated( + "3.28.0", + "4.0.0", + addendum="Use `conda_build.os_utils.liefldd.codefile_class` instead.", +) +def codefile_type_liefldd(*args, **kwargs) -> str | None: + codefile = codefile_class(*args, **kwargs) + return codefile.__name__ if codefile else None + + +deprecated.constant( + "3.28.0", + "4.0.0", + "codefile_type_pyldd", + _codefile_type, + addendum="Use `conda_build.os_utils.pyldd.codefile_class` instead.", +) +deprecated.constant( + "3.28.0", + "4.0.0", + "codefile_type", + _codefile_type, + addendum="Use `conda_build.os_utils.liefldd.codefile_class` instead.", +) def _trim_sysroot(sysroot): @@ -111,7 +139,9 @@ def get_libraries(file): if binary.format == lief.EXE_FORMATS.PE: result = binary.libraries else: - result = [lib if is_string(lib) else lib.name for lib in binary.libraries] + result = [ + lib if isinstance(lib, str) else lib.name for lib in binary.libraries + ] # LIEF returns LC_ID_DYLIB name @rpath/libbz2.dylib in binary.libraries. Strip that. binary_name = None if binary.format == lief.EXE_FORMATS.MACHO: @@ -505,7 +535,7 @@ def inspect_linkages_lief( while tmp_filename: if ( not parent_exe_dirname - and codefile_type(tmp_filename) == "EXEfile" + and codefile_class(tmp_filename) == EXEfile ): parent_exe_dirname = os.path.dirname(tmp_filename) tmp_filename = parents_by_filename[tmp_filename] @@ -595,7 +625,7 @@ def get_linkages( result_pyldd = [] debug = False if not have_lief or debug: - if codefile_type(filename) not in ("DLLfile", "EXEfile"): + if codefile_class(filename) not in (DLLfile, EXEfile): result_pyldd = inspect_linkages_pyldd( filename, resolve_filenames=resolve_filenames, @@ -607,7 +637,7 @@ def get_linkages( return result_pyldd else: print( - f"WARNING: failed to get_linkages, codefile_type('{filename}')={codefile_type(filename)}" + f"WARNING: failed to get_linkages, codefile_class('{filename}')={codefile_class(filename)}" ) return {} result_lief = inspect_linkages_lief( diff --git a/conda_build/os_utils/pyldd.py b/conda_build/os_utils/pyldd.py index 1e1cd4e4cc..42b89711ae 100644 --- a/conda_build/os_utils/pyldd.py +++ b/conda_build/os_utils/pyldd.py @@ -1,5 +1,7 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + import argparse import glob import logging @@ -7,9 +9,12 @@ import re import struct import sys +from pathlib import Path from conda_build.utils import ensure_list, get_logger +from ..deprecations import deprecated + logging.basicConfig(level=logging.INFO) @@ -1028,46 +1033,60 @@ def codefile(file, arch="any", initial_rpaths_transitive=[]): return inscrutablefile(file, list(initial_rpaths_transitive)) -def codefile_class(filename, skip_symlinks=False): - if os.path.islink(filename): - if skip_symlinks: - return None - else: - filename = os.path.realpath(filename) - if os.path.isdir(filename): +def codefile_class( + path: str | os.PathLike | Path, + skip_symlinks: bool = False, +) -> type[DLLfile | EXEfile | machofile | elffile] | None: + # same signature as conda.os_utils.liefldd.codefile_class + path = Path(path) + if skip_symlinks and path.is_symlink(): return None - if filename.endswith((".dll", ".pyd")): + path = path.resolve() + + def _get_magic_bit(path: Path) -> bytes: + with path.open("rb") as handle: + bit = handle.read(4) + return struct.unpack(BIG_ENDIAN + "L", bit)[0] + + if path.is_dir(): + return None + elif path.suffix.lower() in (".dll", ".pyd"): return DLLfile - if filename.endswith(".exe"): + elif path.suffix.lower() == ".exe": return EXEfile - # Java .class files share 0xCAFEBABE with Mach-O FAT_MAGIC. - if filename.endswith(".class"): + elif path.suffix.lower() == ".class": + # Java .class files share 0xCAFEBABE with Mach-O FAT_MAGIC. return None - if not os.path.exists(filename) or os.path.getsize(filename) < 4: + elif not path.exists() or path.stat().st_size < 4: + return None + elif (magic := _get_magic_bit(path)) == ELF_HDR: + return elffile + elif magic in (FAT_MAGIC, MH_MAGIC, MH_CIGAM, MH_CIGAM_64): + return machofile + else: return None - with open(filename, "rb") as file: - (magic,) = struct.unpack(BIG_ENDIAN + "L", file.read(4)) - file.seek(0) - if magic in (FAT_MAGIC, MH_MAGIC, MH_CIGAM, MH_CIGAM_64): - return machofile - elif magic == ELF_HDR: - return elffile - return None -def is_codefile(filename, skip_symlinks=True): - klass = codefile_class(filename, skip_symlinks=skip_symlinks) - if not klass: - return False - return True +@deprecated( + "3.28.0", + "4.0.0", + addendum="Use `conda_build.os_utils.pyldd.codefile_class` instead.", +) +def is_codefile(path: str | os.PathLike | Path, skip_symlinks: bool = True) -> bool: + return bool(codefile_class(path, skip_symlinks=skip_symlinks)) -def codefile_type(filename, skip_symlinks=True): - "Returns None, 'machofile' or 'elffile'" - klass = codefile_class(filename, skip_symlinks=skip_symlinks) - if not klass: - return None - return klass.__name__ +@deprecated( + "3.28.0", + "4.0.0", + addendum="Use `conda_build.os_utils.pyldd.codefile_class` instead.", +) +def codefile_type( + path: str | os.PathLike | Path, + skip_symlinks: bool = True, +) -> str | None: + codefile = codefile_class(path, skip_symlinks=skip_symlinks) + return codefile.__name__ if codefile else None def _trim_sysroot(sysroot): diff --git a/conda_build/post.py b/conda_build/post.py index 290779385d..05af50b24f 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -53,18 +53,24 @@ have_lief, set_rpath, ) -from conda_build.os_utils.pyldd import codefile_type +from conda_build.os_utils.pyldd import ( + DLLfile, + EXEfile, + codefile_class, + elffile, + machofile, +) filetypes_for_platform = { - "win": ("DLLfile", "EXEfile"), - "osx": ["machofile"], - "linux": ["elffile"], + "win": (DLLfile, EXEfile), + "osx": (machofile,), + "linux": (elffile,), } def fix_shebang(f, prefix, build_python, osx_is_app=False): path = join(prefix, f) - if codefile_type(path): + if codefile_class(path): return elif islink(path): return @@ -405,7 +411,7 @@ def osx_ch_link(path, link_dict, host_prefix, build_prefix, files): ".. seems to be linking to a compiler runtime, replacing build prefix with " "host prefix and" ) - if not codefile_type(link): + if not codefile_class(link): sys.exit( "Error: Compiler runtime library in build prefix not found in host prefix %s" % link @@ -841,7 +847,7 @@ def _collect_needed_dsos( sysroots = list(sysroots_files.keys())[0] for f in files: path = join(run_prefix, f) - if not codefile_type(path): + if not codefile_class(path): continue build_prefix = build_prefix.replace(os.sep, "/") run_prefix = run_prefix.replace(os.sep, "/") @@ -901,10 +907,9 @@ def _map_file_to_package( for subdir2, _, filez in os.walk(prefix): for file in filez: fp = join(subdir2, file) - dynamic_lib = ( - any(fnmatch(fp, ext) for ext in ("*.so*", "*.dylib*", "*.dll")) - and codefile_type(fp, skip_symlinks=False) is not None - ) + dynamic_lib = any( + fnmatch(fp, ext) for ext in ("*.so*", "*.dylib*", "*.dll") + ) and codefile_class(fp, skip_symlinks=False) static_lib = any(fnmatch(fp, ext) for ext in ("*.a", "*.lib")) # Looking at all the files is very slow. if not dynamic_lib and not static_lib: @@ -947,7 +952,7 @@ def _map_file_to_package( ) } all_lib_exports[prefix][rp_po] = exports - # Check codefile_type to filter out linker scripts. + # Check codefile_class to filter out linker scripts. if dynamic_lib: contains_dsos[prefix_owners[prefix][rp_po][0]] = True elif static_lib: @@ -1217,8 +1222,8 @@ def _show_linking_messages( ) for f in files: path = join(run_prefix, f) - filetype = codefile_type(path) - if not filetype or filetype not in filetypes_for_platform[subdir.split("-")[0]]: + codefile = codefile_class(path) + if codefile not in filetypes_for_platform[subdir.split("-")[0]]: continue warn_prelude = "WARNING ({},{})".format(pkg_name, f.replace(os.sep, "/")) err_prelude = " ERROR ({},{})".format(pkg_name, f.replace(os.sep, "/")) @@ -1316,15 +1321,15 @@ def check_overlinking_impl( files_to_inspect = [] filesu = [] - for f in files: - path = join(run_prefix, f) - filetype = codefile_type(path) - if filetype and filetype in filetypes_for_platform[subdir.split("-")[0]]: - files_to_inspect.append(f) - filesu.append(f.replace("\\", "/")) + for file in files: + path = join(run_prefix, file) + codefile = codefile_class(path) + if codefile in filetypes_for_platform[subdir.split("-")[0]]: + files_to_inspect.append(file) + filesu.append(file.replace("\\", "/")) if not files_to_inspect: - return dict() + return {} sysroot_substitution = "$SYSROOT" build_prefix_substitution = "$PATH" @@ -1633,18 +1638,18 @@ def post_process_shared_lib(m, f, files, host_prefix=None): if not host_prefix: host_prefix = m.config.host_prefix path = join(host_prefix, f) - codefile_t = codefile_type(path) - if not codefile_t or path.endswith(".debug"): + codefile = codefile_class(path) + if not codefile or path.endswith(".debug"): return rpaths = m.get_value("build/rpaths", ["lib"]) - if codefile_t == "elffile": + if codefile == elffile: mk_relative_linux( f, host_prefix, rpaths=rpaths, method=m.get_value("build/rpaths_patcher", None), ) - elif codefile_t == "machofile": + elif codefile == machofile: if m.config.host_platform != "osx": log = utils.get_logger(__name__) log.warn( @@ -1734,7 +1739,7 @@ def check_symlinks(files, prefix, croot): # symlinks to binaries outside of the same dir don't work. RPATH stuff gets confused # because ld.so follows symlinks in RPATHS # If condition exists, then copy the file rather than symlink it. - if not dirname(link_path) == dirname(real_link_path) and codefile_type(f): + if not dirname(link_path) == dirname(real_link_path) and codefile_class(f): os.remove(path) utils.copy_into(real_link_path, path) elif real_link_path.startswith(real_build_prefix): diff --git a/conda_build/utils.py b/conda_build/utils.py index 06a5c79c6d..f8606ffe9d 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -97,9 +97,10 @@ FileNotFoundError = FileNotFoundError on_win = sys.platform == "win32" +on_mac = sys.platform == "darwin" +on_linux = sys.platform == "linux" codec = getpreferredencoding() or "utf-8" -on_win = sys.platform == "win32" root_script_dir = os.path.join(root_dir, "Scripts" if on_win else "bin") mmap_MAP_PRIVATE = 0 if on_win else mmap.MAP_PRIVATE mmap_PROT_READ = 0 if on_win else mmap.PROT_READ diff --git a/news/5040-codefile b/news/5040-codefile new file mode 100644 index 0000000000..c4f85ca7cf --- /dev/null +++ b/news/5040-codefile @@ -0,0 +1,21 @@ +### Enhancements + +* + +### Bug fixes + +* + +### Deprecations + +* Mark `conda_build.os_utils.pyldd.is_string` as pending deprecation. Use `isinstance(value, str)` instead. (#5040) +* Mark `conda_build.os_utils.pyldd.is_codefile` as pending deprecation. Use `conda_build.os_utils.pyldd.codefile_class` instead. (#5040) +* Mark `conda_build.os_utils.pyldd.codefile_type` as pending deprecation. Use `conda_build.os_utils.pyldd.codefile_class` instead. (#5040) + +### Docs + +* + +### Other + +* diff --git a/tests/data/ldd/clear.elf b/tests/data/ldd/clear.elf new file mode 100755 index 0000000000..52013aa3ee Binary files /dev/null and b/tests/data/ldd/clear.elf differ diff --git a/tests/data/ldd/clear.exe b/tests/data/ldd/clear.exe new file mode 100644 index 0000000000..bd7543feba Binary files /dev/null and b/tests/data/ldd/clear.exe differ diff --git a/tests/data/ldd/clear.macho b/tests/data/ldd/clear.macho new file mode 100755 index 0000000000..8de24d5608 Binary files /dev/null and b/tests/data/ldd/clear.macho differ diff --git a/tests/data/ldd/jansi.dll b/tests/data/ldd/jansi.dll new file mode 100755 index 0000000000..81433eef25 Binary files /dev/null and b/tests/data/ldd/jansi.dll differ diff --git a/tests/data/ldd/uuid.pyd b/tests/data/ldd/uuid.pyd new file mode 100644 index 0000000000..f99ad10b9b Binary files /dev/null and b/tests/data/ldd/uuid.pyd differ diff --git a/tests/os_utils/test_codefile.py b/tests/os_utils/test_codefile.py new file mode 100644 index 0000000000..a3e38342da --- /dev/null +++ b/tests/os_utils/test_codefile.py @@ -0,0 +1,40 @@ +# Copyright (C) 2014 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +import pytest + +from conda_build.os_utils.liefldd import codefile_class as liefldd_codefile_class +from conda_build.os_utils.pyldd import DLLfile, EXEfile, elffile, machofile +from conda_build.os_utils.pyldd import codefile_class as pyldd_codefile_class + +LDD = Path(__file__).parent.parent / "data" / "ldd" + + +@pytest.mark.parametrize( + "path,expect", + [ + pytest.param(__file__, None, id="Unknown"), + pytest.param(LDD / "jansi.dll", DLLfile, id="DLL"), + pytest.param(LDD / "uuid.pyd", DLLfile, id="PYD"), + pytest.param(LDD / "clear.exe", EXEfile, id="EXE"), + pytest.param(LDD / "clear.macho", machofile, id="MACHO"), + pytest.param(LDD / "clear.elf", elffile, id="ELF"), + ], +) +@pytest.mark.parametrize( + "codefile_class", + [ + pytest.param(pyldd_codefile_class, id="pyldd"), + pytest.param(liefldd_codefile_class, id="liefldd"), + ], +) +def test_codefile_class( + path: str | Path, + expect: type[DLLfile | EXEfile | machofile | elffile] | None, + codefile_class: Callable, +): + assert codefile_class(path) == expect