diff --git a/craft_parts/executor/part_handler.py b/craft_parts/executor/part_handler.py index dc67939bc..f7ddd0a94 100644 --- a/craft_parts/executor/part_handler.py +++ b/craft_parts/executor/part_handler.py @@ -285,7 +285,7 @@ def _run_build( :return: The build step state. """ self._make_dirs() - self._unpack_stage_packages() + self._unpack_stage_packages() # NOTE: this is where stage-packages are xattr annotated self._unpack_stage_snaps() if not update and not self._plugin.get_out_of_source_build(): @@ -315,6 +315,9 @@ def _run_build( stderr=stderr, ) + # The plugin has by now generated our list of packages (_run_step -> run_builtin -> _builtin_build), now parse it to get the list of files and xattr them + self._annotate_plugin_files() + # Organize the installed files as requested. We do this in the build step for # two reasons: # @@ -982,6 +985,22 @@ def _unpack_stage_snaps(self) -> None: for snap_source in snap_sources: snap_source.provision(install_dir, keep=True) + def _annotate_plugin_files(self) -> None: + """If the plugin has generated a list of the files it is staging, get that list and annotate those files.""" + got_something = False + for pkg_name, pkg_files in self._plugin.get_file_list().items(): + print("******************************************************") + print(pkg_name) + print("******************************************************") + for f in pkg_files: + print("-", f.as_posix()) + got_something = True + #if got_something: + #breakpoint() + #import sys + #sys.exit() + + # TODO: Iterate over primed files and call mark_origin_stage_package def _remove(filename: Path) -> None: """Remove the given directory entry. diff --git a/craft_parts/executor/step_handler.py b/craft_parts/executor/step_handler.py index aee896592..a4e5291bc 100644 --- a/craft_parts/executor/step_handler.py +++ b/craft_parts/executor/step_handler.py @@ -153,6 +153,7 @@ def _builtin_build(self) -> StepContents: stdout=self._stdout, stderr=self._stderr, ) + except subprocess.CalledProcessError as process_error: raise errors.PluginBuildError( part_name=self._part.name, plugin_name=self._part.plugin_name diff --git a/craft_parts/plugins/base.py b/craft_parts/plugins/base.py index 02ab54913..6ee547336 100644 --- a/craft_parts/plugins/base.py +++ b/craft_parts/plugins/base.py @@ -84,6 +84,13 @@ def get_out_of_source_build(cls) -> bool: def get_build_commands(self) -> list[str]: """Return a list of commands to run during the build step.""" + def get_file_list(self) -> dict[str, list[Path]]: + """Return a mapping of installed package name -> file list, if the plugin supports it. + + Used for xattr annotation. + """ + return {} + def set_action_properties(self, action_properties: ActionProperties) -> None: """Store a copy of the given action properties. @@ -136,6 +143,11 @@ def _get_venv_directory(self) -> pathlib.Path: """ return self._part_info.part_install_dir + def _get_site_packages_directory(self) -> pathlib.Path: + """Get the directory into which site packages are installed in the venv.""" + venvdir = self._get_venv_directory() + return venvdir / "lib" / pyver / "site-packages" + def _get_create_venv_commands(self) -> list[str]: """Get the commands for setting up the virtual environment.""" venv_dir = self._get_venv_directory() diff --git a/craft_parts/plugins/npm_plugin.py b/craft_parts/plugins/npm_plugin.py index 465417dca..d09f9b38e 100644 --- a/craft_parts/plugins/npm_plugin.py +++ b/craft_parts/plugins/npm_plugin.py @@ -16,8 +16,10 @@ """The npm plugin.""" +import json import logging import os +from pathlib import Path import platform import re from textwrap import dedent @@ -320,3 +322,99 @@ def get_build_commands(self) -> list[str]: ) ] return cmd + + def _append_package_dir(self, pkg_dir: Path, pkg_name: str | None, file_list: dict[str, list[Path]], missing_deps: list[str]) -> None: + """Reads a package dir and appends to file_list. + + :param pkg_dir: A dir containing a package.json. + :param pkg_name: Passed when the package's name is known (i.e., not at the top level). + :param file_list: The data structure to add the package to. + :param missing_deps: A list of package name dependencies that couldn't be found via simple resolution, for verifying at the end. + """ + if pkg_dir.name.startswith("@"): + # This is a scoped name: the package.json file(s) will be in dir(s) under the scope name dir. + # https://docs.npmjs.com/cli/v9/using-npm/scope + missing_deps = [] + for unscoped_pkg_dir in pkg_dir.iterdir(): + # TODO: Do we need to build the name to pass in the recursive call here? + self._append_package_dir(unscoped_pkg_dir, None, file_list, missing_deps) + return + + # From the package-json npm docs page: + # "If the package.json for your package is not in the root directory (for example if it is part of a monorepo), you can specify the directory in which it lives" + # Need to figure out whether the final package will always have a package.json on the top level, even if the source repo doesn't. + + metadata_file = pkg_dir / "package.json" + try: + with open(metadata_file, "r") as f: + metadata = json.load(f) + except FileNotFoundError: + # The package.json didn't exist, assume this isn't a top-level dir and this means that we have dependencies that are at the same level as each other. For instance, if A depends on B and C, and B also depends on C, then we'll fail to find C under B's node_modules dir because it already exists in A's node_modules dir. + # If the package name hasn't been passed, then something is wrong. + if not pkg_name: + raise + missing_deps.append(pkg_name) + return + + # https://docs.npmjs.com/cli/v10/configuring-npm/package-json + + pkg_name = metadata["name"] + if pkg_name in file_list: + # We've already resolved this package at a different level + return + + # NPM allows str or dict for bin - normalize it + bintype = type(metadata.get("bin", False)) + bins = [] + if bintype is str: + bins = [pkg_dir / metadata["bin"]] + elif bintype is dict: + bins = [pkg_dir / bin_file for _, bin_file in metadata["bin"].items()] + + # TODO: man + + # TODO: directories (directories.bin, directories.man) + + # TODO: scripts + + # TODO: workspaces? + + # TODO: verify default values + + file_list[pkg_name] = [ + pkg_dir / metadata.get("main", "index.js"), + + ] + bins + + # TODO: peerDependencies + # (may or may not be installed) + + # TODO: bundleDependencies + # (do we want to call out files that were bundled in as originating from their actual source package?) + + # TODO: optionalDepenencies + + try: + for dep_name in metadata.get("dependencies", []): + dep_pkg_dir = pkg_dir / "node_modules" / dep_name + self._append_package_dir(dep_pkg_dir, dep_name, file_list, missing_deps) + except KeyError as e: + # No dependencies + pass + + + # TODO: It seems like node packages don't have a manifest? In which case we'll need to inspect the archive, and thus have a plugin hook at a different spot + + def get_file_list(self) -> dict[str, list[Path]]: + modules_dir = self._part_info.part_install_dir / "usr/lib/node_modules" + + ret = {} + missing = [] + for pkg_dir in modules_dir.iterdir(): + # These pkg_dirs will be the leaf modules installed by the user + self._append_package_dir(pkg_dir, None, ret, missing) + #breakpoint() + + # TODO: Figure out if the "missing packages" is actually a case we can hit, and either add in the checking for it or remove it + + return ret diff --git a/craft_parts/plugins/python_plugin.py b/craft_parts/plugins/python_plugin.py index b7b170397..5daa9e689 100644 --- a/craft_parts/plugins/python_plugin.py +++ b/craft_parts/plugins/python_plugin.py @@ -16,6 +16,12 @@ """The python plugin.""" +import csv +from email import policy +from email.parser import HeaderParser +import json +from pathlib import Path +import re import shlex from typing import Literal @@ -23,6 +29,10 @@ from .properties import PluginProperties +_TEST_DIR_INSTALLABLE_CMD = "[ -f setup.py ] || [ -f pyproject.toml ]" +_PIP_SHOW_OUT_FILE = "pipshow.txt.eml" + + class PythonPluginProperties(PluginProperties, frozen=True): """The part properties used by the python plugin.""" @@ -31,11 +41,44 @@ class PythonPluginProperties(PluginProperties, frozen=True): python_requirements: list[str] = [] python_constraints: list[str] = [] python_packages: list[str] = ["pip", "setuptools", "wheel"] + + resolved_installed_packages: dict[str, Path] = {} # part properties required by the plugin source: str # pyright: ignore[reportGeneralTypeIssues] + def get_formatted_constraints(self) -> str: + """Return a string ready to be passed as part of a pip command line. + + For instance: "-c constraints.txt -c constraints-dev.txt" + """ + if not self.python_constraints: + return "" + return " ".join( + f"-c {c!r}" for c in self.python_constraints + ) + + def get_formatted_requirements(self) -> str: + """Return a string ready to be passed as part of a pip command line. + + For instance: "-r requirements.txt -r requirements-dev.txt" + """ + return " ".join( + f"-r {r!r}" for r in self.python_requirements + ) + + + def get_formatted_packages(self) -> str: + """Return a string ready to be passed as part of a pip command line. + + For instance: "'flask' 'requests'" + """ + return " ".join( + [shlex.quote(pkg) for pkg in self.python_packages] + ) + + class PythonPlugin(BasePythonPlugin): """A plugin to build python parts.""" @@ -43,33 +86,61 @@ class PythonPlugin(BasePythonPlugin): _options: PythonPluginProperties def _get_package_install_commands(self) -> list[str]: - commands = [] + install_commands = [] pip = self._get_pip() - - if self._options.python_constraints: - constraints = " ".join( - f"-c {c!r}" for c in self._options.python_constraints - ) - else: - constraints = "" + constraints = self._options.get_formatted_constraints() if self._options.python_packages: - python_packages = " ".join( - [shlex.quote(pkg) for pkg in self._options.python_packages] - ) + python_packages = self._options.get_formatted_packages() python_packages_cmd = f"{pip} install {constraints} -U {python_packages}" - commands.append(python_packages_cmd) + install_commands.append(python_packages_cmd) if self._options.python_requirements: - requirements = " ".join( - f"-r {r!r}" for r in self._options.python_requirements - ) + requirements = self._options.get_formatted_requirements() requirements_cmd = f"{pip} install {constraints} -U {requirements}" - commands.append(requirements_cmd) + install_commands.append(requirements_cmd) - commands.append( - f"[ -f setup.py ] || [ -f pyproject.toml ] && {pip} install {constraints} -U ." + install_commands.append( + f"{_TEST_DIR_INSTALLABLE_CMD} && {pip} install {constraints} -U ." ) - return commands + return install_commands + + def get_file_list(self) -> dict[str, list[Path]]: + # https://packaging.python.org/en/latest/specifications/binary-distribution-format/ + # Read the RECORD files + + venvdir = self._get_venv_directory() + python_path = venvdir / "bin/python" + python_version = python_path.resolve().name + + site_pkgs_dir = venvdir / "lib" / python_version / "site-packages" + + # Could also add the pkginfo library and skip a lot of the below + + ret = {} + for pkg_dir in site_pkgs_dir.iterdir(): + # We only care about the metadata dirs + if not pkg_dir.name.endswith(".dist-info"): + continue + + # Get package name from filename - could also parse it from METADATA + # I assume there must be at least one character for version, not sure what else it could look like though. I've seen: + # - 0.0.0 + # - 0.0 + # TODO: look this up + pkg_name_match = re.match(r"^(.*)-[0-9.]+\.dist-info$", pkg_dir.name) + if not pkg_name_match: + raise Exception(f"Unexpectedly formatted dist-info dir: {pkg_dir.name!r}") + pkg_name = pkg_name_match[1] + + record_file = pkg_dir / "RECORD" + with open(record_file, "r") as f: + csvreader = csv.reader(f) + + # First row is files + # TODO: Remove all files listed under the dist-info dir? + pkg_files = [Path(f[0]) for f in csvreader] + ret[pkg_name] = pkg_files + return ret