Artifact tagging prototype

craft_parts/executor/part_handler.py

@@ -285,7 +285,7 @@ def _run_build(
         :return: The build step state.
         """
         self._make_dirs()
-        self._unpack_stage_packages()
+        self._unpack_stage_packages()  # NOTE: this is where stage-packages are xattr annotated
         self._unpack_stage_snaps()
 
         if not update and not self._plugin.get_out_of_source_build():
@@ -315,6 +315,9 @@ def _run_build(
                 stderr=stderr,
             )
 
+        # The plugin has by now generated our list of packages (_run_step -> run_builtin -> _builtin_build), now parse it to get the list of files and xattr them
+        self._annotate_plugin_files()
+
         # Organize the installed files as requested. We do this in the build step for
         # two reasons:
         #
@@ -982,6 +985,22 @@ def _unpack_stage_snaps(self) -> None:
         for snap_source in snap_sources:
             snap_source.provision(install_dir, keep=True)
 
+    def _annotate_plugin_files(self) -> None:
+        """If the plugin has generated a list of the files it is staging, get that list and annotate those files."""
+        got_something = False
+        for pkg_name, pkg_files in self._plugin.get_file_list().items():
+            print("******************************************************")
+            print(pkg_name)
+            print("******************************************************")
+            for f in pkg_files:
+                print("-", f.as_posix())
+            got_something = True
+        #if got_something:
+            #breakpoint()
+            #import sys
+            #sys.exit()
+
+        # TODO: Iterate over primed files and call mark_origin_stage_package 
 
 def _remove(filename: Path) -> None:
     """Remove the given directory entry.

craft_parts/executor/step_handler.py

@@ -153,6 +153,7 @@ def _builtin_build(self) -> StepContents:
                 stdout=self._stdout,
                 stderr=self._stderr,
             )
+
         except subprocess.CalledProcessError as process_error:
             raise errors.PluginBuildError(
                 part_name=self._part.name, plugin_name=self._part.plugin_name

craft_parts/plugins/base.py

@@ -84,6 +84,13 @@ def get_out_of_source_build(cls) -> bool:
     def get_build_commands(self) -> list[str]:
         """Return a list of commands to run during the build step."""
 
+    def get_file_list(self) -> dict[str, list[Path]]:
+        """Return a mapping of installed package name -> file list, if the plugin supports it.
+
+        Used for xattr annotation.
+        """
+        return {}
+
     def set_action_properties(self, action_properties: ActionProperties) -> None:
         """Store a copy of the given action properties.
 
@@ -136,6 +143,11 @@ def _get_venv_directory(self) -> pathlib.Path:
         """
         return self._part_info.part_install_dir
 
+    def _get_site_packages_directory(self) -> pathlib.Path:
+        """Get the directory into which site packages are installed in the venv."""
+        venvdir = self._get_venv_directory()
+        return venvdir / "lib" / pyver / "site-packages"
+
     def _get_create_venv_commands(self) -> list[str]:
         """Get the commands for setting up the virtual environment."""
         venv_dir = self._get_venv_directory()

craft_parts/plugins/npm_plugin.py

@@ -16,8 +16,10 @@
 
 """The npm plugin."""
 
+import json
 import logging
 import os
+from pathlib import Path
 import platform
 import re
 from textwrap import dedent
@@ -320,3 +322,99 @@ def get_build_commands(self) -> list[str]:
             )
         ]
         return cmd
+
+    def _append_package_dir(self, pkg_dir: Path, pkg_name: str | None, file_list: dict[str, list[Path]], missing_deps: list[str]) -> None:
+        """Reads a package dir and appends to file_list.
+
+        :param pkg_dir: A dir containing a package.json.
+        :param pkg_name: Passed when the package's name is known (i.e., not at the top level).
+        :param file_list: The data structure to add the package to.
+        :param missing_deps: A list of package name dependencies that couldn't be found via simple resolution, for verifying at the end.
+        """
+        if pkg_dir.name.startswith("@"):
+            # This is a scoped name: the package.json file(s) will be in dir(s) under the scope name dir.
+            # https://docs.npmjs.com/cli/v9/using-npm/scope
+            missing_deps = []
+            for unscoped_pkg_dir in pkg_dir.iterdir():
+                # TODO: Do we need to build the name to pass in the recursive call here?
+                self._append_package_dir(unscoped_pkg_dir, None, file_list, missing_deps)
+            return
+
+        # From the package-json npm docs page:
+        # "If the package.json for your package is not in the root directory (for example if it is part of a monorepo), you can specify the directory in which it lives"
+        # Need to figure out whether the final package will always have a package.json on the top level, even if the source repo doesn't.
+
+        metadata_file = pkg_dir / "package.json"
+        try:
+            with open(metadata_file, "r") as f:
+                metadata = json.load(f)
+        except FileNotFoundError:
+            # The package.json didn't exist, assume this isn't a top-level dir and this means that we have dependencies that are at the same level as each other.  For instance, if A depends on B and C, and B also depends on C, then we'll fail to find C under B's node_modules dir because it already exists in A's node_modules dir.
+            # If the package name hasn't been passed, then something is wrong.
+            if not pkg_name:
+                raise
+            missing_deps.append(pkg_name)
+            return
+
+        # https://docs.npmjs.com/cli/v10/configuring-npm/package-json
+
+        pkg_name = metadata["name"]
+        if pkg_name in file_list:
+            # We've already resolved this package at a different level
+            return
+
+        # NPM allows str or dict for bin - normalize it
+        bintype = type(metadata.get("bin", False))
+        bins = []
+        if bintype is str:
+            bins = [pkg_dir / metadata["bin"]]
+        elif bintype is dict:
+            bins = [pkg_dir / bin_file for _, bin_file in metadata["bin"].items()]
+
+        # TODO: man
+
+        # TODO: directories (directories.bin, directories.man)
+
+        # TODO: scripts
+
+        # TODO: workspaces?
+
+        # TODO: verify default values
+
+        file_list[pkg_name] = [
+            pkg_dir / metadata.get("main", "index.js"),
+
+        ] + bins
+
+        # TODO: peerDependencies
+        # (may or may not be installed)
+
+        # TODO: bundleDependencies
+        # (do we want to call out files that were bundled in as originating from their actual source package?)
+
+        # TODO: optionalDepenencies
+
+        try:
+            for dep_name in metadata.get("dependencies", []):
+                dep_pkg_dir = pkg_dir / "node_modules" / dep_name
+                self._append_package_dir(dep_pkg_dir, dep_name, file_list, missing_deps)
+        except KeyError as e:
+            # No dependencies
+            pass
+
+
+    # TODO: It seems like node packages don't have a manifest?  In which case we'll need to inspect the archive, and thus have a plugin hook at a different spot
+
+    def get_file_list(self) -> dict[str, list[Path]]:
+        modules_dir = self._part_info.part_install_dir / "usr/lib/node_modules"
+
+        ret = {}
+        missing = []
+        for pkg_dir in modules_dir.iterdir():
+            # These pkg_dirs will be the leaf modules installed by the user
+            self._append_package_dir(pkg_dir, None, ret, missing)
+        #breakpoint()
+
+        # TODO: Figure out if the "missing packages" is actually a case we can hit, and either add in the checking for it or remove it
+
+        return ret

craft_parts/plugins/python_plugin.py

@@ -16,13 +16,23 @@
 
 """The python plugin."""
 
+import csv
+from email import policy
+from email.parser import HeaderParser
+import json
+from pathlib import Path
+import re
 import shlex
 from typing import Literal
 
 from .base import BasePythonPlugin
 from .properties import PluginProperties
 
 
+_TEST_DIR_INSTALLABLE_CMD = "[ -f setup.py ] || [ -f pyproject.toml ]"
+_PIP_SHOW_OUT_FILE = "pipshow.txt.eml"
+
+
 class PythonPluginProperties(PluginProperties, frozen=True):
     """The part properties used by the python plugin."""
 
@@ -31,45 +41,106 @@ class PythonPluginProperties(PluginProperties, frozen=True):
     python_requirements: list[str] = []
     python_constraints: list[str] = []
     python_packages: list[str] = ["pip", "setuptools", "wheel"]
+
+    resolved_installed_packages: dict[str, Path] = {}
 
     # part properties required by the plugin
     source: str  # pyright: ignore[reportGeneralTypeIssues]
 
 
+    def get_formatted_constraints(self) -> str:
+        """Return a string ready to be passed as part of a pip command line.
+
+        For instance: "-c constraints.txt -c constraints-dev.txt"
+        """
+        if not self.python_constraints:
+            return ""
+        return " ".join(
+            f"-c {c!r}" for c in self.python_constraints
+        )
+
+    def get_formatted_requirements(self) -> str:
+        """Return a string ready to be passed as part of a pip command line.
+
+        For instance: "-r requirements.txt -r requirements-dev.txt"
+        """
+        return " ".join(
+            f"-r {r!r}" for r in self.python_requirements
+        )
+
+
+    def get_formatted_packages(self) -> str:
+        """Return a string ready to be passed as part of a pip command line.
+
+        For instance: "'flask' 'requests'"
+        """
+        return " ".join(
+            [shlex.quote(pkg) for pkg in self.python_packages]
+        )
+
+
 class PythonPlugin(BasePythonPlugin):
     """A plugin to build python parts."""
 
     properties_class = PythonPluginProperties
     _options: PythonPluginProperties
 
     def _get_package_install_commands(self) -> list[str]:
-        commands = []
+        install_commands = []
 
         pip = self._get_pip()
-
-        if self._options.python_constraints:
-            constraints = " ".join(
-                f"-c {c!r}" for c in self._options.python_constraints
-            )
-        else:
-            constraints = ""
+        constraints = self._options.get_formatted_constraints()
 
         if self._options.python_packages:
-            python_packages = " ".join(
-                [shlex.quote(pkg) for pkg in self._options.python_packages]
-            )
+            python_packages = self._options.get_formatted_packages()
             python_packages_cmd = f"{pip} install {constraints} -U {python_packages}"
-            commands.append(python_packages_cmd)
+            install_commands.append(python_packages_cmd)
 
         if self._options.python_requirements:
-            requirements = " ".join(
-                f"-r {r!r}" for r in self._options.python_requirements
-            )
+            requirements = self._options.get_formatted_requirements()
             requirements_cmd = f"{pip} install {constraints} -U {requirements}"
-            commands.append(requirements_cmd)
+            install_commands.append(requirements_cmd)
 
-        commands.append(
-            f"[ -f setup.py ] || [ -f pyproject.toml ] && {pip} install {constraints} -U ."
+        install_commands.append(
+            f"{_TEST_DIR_INSTALLABLE_CMD} && {pip} install {constraints} -U ."
         )
 
-        return commands
+        return install_commands
+
+    def get_file_list(self) -> dict[str, list[Path]]:
+        # https://packaging.python.org/en/latest/specifications/binary-distribution-format/
+        # Read the RECORD files
+
+        venvdir = self._get_venv_directory()
+        python_path = venvdir / "bin/python"
+        python_version = python_path.resolve().name
+
+        site_pkgs_dir = venvdir / "lib" / python_version / "site-packages"
+
+        # Could also add the pkginfo library and skip a lot of the below
+
+        ret = {}
+        for pkg_dir in site_pkgs_dir.iterdir():
+            # We only care about the metadata dirs
+            if not pkg_dir.name.endswith(".dist-info"):
+                continue
+
+            # Get package name from filename - could also parse it from METADATA
+            # I assume there must be at least one character for version, not sure what else it could look like though.  I've seen:
+            # - 0.0.0
+            # - 0.0
+            # TODO: look this up
+            pkg_name_match = re.match(r"^(.*)-[0-9.]+\.dist-info$", pkg_dir.name)
+            if not pkg_name_match:
+                raise Exception(f"Unexpectedly formatted dist-info dir: {pkg_dir.name!r}")
+            pkg_name = pkg_name_match[1]
+
+            record_file = pkg_dir / "RECORD"
+            with open(record_file, "r") as f:
+                csvreader = csv.reader(f)
+
+                # First row is files
+                # TODO: Remove all files listed under the dist-info dir?
+                pkg_files = [Path(f[0]) for f in csvreader]
+                ret[pkg_name] = pkg_files
+        return ret