From b2c092136f6c81eb22d497017c99b6829d35154a Mon Sep 17 00:00:00 2001 From: Thorsten Beier Date: Fri, 5 Jan 2024 09:28:33 +0100 Subject: [PATCH] remove prefix bundler abstraction and use empackindirections (#11) --- jupyterlite_xeus/add_on.py | 93 ++++++++++++------- jupyterlite_xeus/prefix_bundler/__init__.py | 21 ----- .../prefix_bundler/empack_bundler.py | 48 ---------- .../prefix_bundler/noop_prefix_bundler.py | 9 -- .../prefix_bundler/prefix_bundler_base.py | 19 ---- 5 files changed, 59 insertions(+), 131 deletions(-) delete mode 100644 jupyterlite_xeus/prefix_bundler/__init__.py delete mode 100644 jupyterlite_xeus/prefix_bundler/empack_bundler.py delete mode 100644 jupyterlite_xeus/prefix_bundler/noop_prefix_bundler.py delete mode 100644 jupyterlite_xeus/prefix_bundler/prefix_bundler_base.py diff --git a/jupyterlite_xeus/add_on.py b/jupyterlite_xeus/add_on.py index 8bb5c15..fc9522a 100644 --- a/jupyterlite_xeus/add_on.py +++ b/jupyterlite_xeus/add_on.py @@ -15,10 +15,12 @@ ) from traitlets import List, Unicode -from .prefix_bundler import get_prefix_bundler from .create_conda_env import create_conda_env_from_yaml,create_conda_env_from_specs from .constants import EXTENSION_NAME, STATIC_DIR +from empack.pack import DEFAULT_CONFIG_PATH, pack_env, pack_directory, add_tarfile_to_env_meta +from empack.file_patterns import pkg_file_filter_from_yaml + def get_kernel_binaries(path): """ return path to the kernel binary (js and wasm) if they exist, else None""" @@ -60,6 +62,12 @@ class XeusAddon(FederatedExtensionAddon): description='The path to the wasm prefix', ) + mounts = List( + [], + config=True, + description="List of paths in the form of : to mount in the wasm prefix", + ) + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.static_dir = self.output_extensions / STATIC_DIR @@ -169,49 +177,66 @@ def copy_kernel(self, kernel_dir, kernel_wasm, kernel_js): (self.copy_one, [kernel_dir / "logo-64x64.png", self.static_dir /"share"/ "jupyter"/ "kernels"/ kernel_dir.name / "logo-64x64.png" ]) ]) + yield from self.pack_prefix(kernel_dir=kernel_dir) - # this part is a bit more complicated: - # Some kernels expect certain files to be at a certain places on the hard drive. - # Ie python (even pure python without additional packages) expects to find certain *.py - # files in a dir like $PREFIX/lib/python3.11/... . - # Since the kernels run in the browser we need a way to take the needed files from the - # $PREFIX of the emscripten-32 wasm env, bundle them into smth like tar.gz file(s) and - # copy them to the static/kernels/ dir. - # - # this concept of taking a prefix and turning it into something the kernels - # can consume is called a "bundler" in this context. - # At the moment, only xpython needs such a bundler, but this is likely to change in the future. - # therefore we do the following. Each kernel can specify which bundler it needs in its kernel.json file. - # If no bundler is specified, we assume that the default bundler is used (which does nothing atm). + def pack_prefix(self, kernel_dir): - language = kernel_spec["language"].lower() - prefix_bundler_name = kernel_spec["metadata"].get("prefix_bundler", None) - prefix_bundler_kwargs = kernel_spec["metadata"].get("prefix_bundler_kwargs", dict()) + kernel_name = kernel_dir.name + packages_dir = Path(self.static_dir) / "share" / "jupyter" / "kernel_packages" + full_kernel_dir = Path(self.static_dir) / "share"/ "jupyter"/"kernels"/ kernel_name + + out_path = Path(self.cwd.name) / "packed_env" + out_path.mkdir(parents=True, exist_ok=True) + # Pack the environment (TODO make this configurable) + file_filters = pkg_file_filter_from_yaml(DEFAULT_CONFIG_PATH) - if language == "python": - # we can also drop the "if" above and just always use empack. - # but this will make the build a bit slower. - # Besides that, there should not be any harm in using empack for all kernels. - # If a kernel does not support empack, it will still just work and will - # **not ** do any extra work at runtime / kernel startup time. - prefix_bundler_name = "empack" + pack_env( + env_prefix=self.prefix, + relocate_prefix="/", + outdir=out_path, + use_cache=True, + file_filters=file_filters + ) - + empack_env_meta = "empack_env_meta.json" + # pack extra dirs + for mount_index,mount in enumerate(self.mounts): + if mount.count(":") != 1: + msg = f"invalid mount {mount}, must be :" + raise ValueError(msg) + host_path, mount_path = mount.split(":") + mount_path = Path(mount_path) + if not mount_path.is_absolute(): + msg = f"mount_path {mount_path} needs to be absolute" + raise ValueError(msg) + outname = f"mount_{mount_index}.tar.gz" + pack_directory(host_dir=host_path, mount_dir=mount_path, + outname=outname, outdir=out_path) + add_tarfile_to_env_meta(env_meta_filename=out_path / empack_env_meta, + tarfile=out_path / outname) + + + # copy all the packages to the packages dir + # (this is shared between multiple xeus-python kernels) + for pkg_path in out_path.iterdir(): + if pkg_path.name.endswith(".tar.gz"): + yield dict( + name=f"xeus:{kernel_name}:copy_package:{pkg_path.name}", + actions=[(self.copy_one, [pkg_path, packages_dir / pkg_path.name ])], + ) - prefix_bundler = get_prefix_bundler( - addon=self, - prefix_bundler_name=prefix_bundler_name, - kernel_name=kernel_dir.name, - **prefix_bundler_kwargs + # copy the empack_env_meta.json + # this is individual for xeus-python kernel + yield dict( + name=f"xeus:{kernel_name}:copy_env_file:{empack_env_meta}", + actions=[(self.copy_one, [out_path / empack_env_meta, Path(full_kernel_dir)/ empack_env_meta ])], ) - - for item in prefix_bundler.build(): - if item: - yield item + + def copy_jupyterlab_extensions_from_prefix(self, manager): # Find the federated extensions in the emscripten-env and install them diff --git a/jupyterlite_xeus/prefix_bundler/__init__.py b/jupyterlite_xeus/prefix_bundler/__init__.py deleted file mode 100644 index 6a91b41..0000000 --- a/jupyterlite_xeus/prefix_bundler/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -from .noop_prefix_bundler import NoopPrefixBundler -from .empack_bundler import EmpackBundler - -# register -prefix_bundler_registry = { - "empack": EmpackBundler, - "default": NoopPrefixBundler # no-op / do nothing -} - -def get_prefix_bundler( - addon, - prefix_bundler_name, - kernel_name, - **kwargs - ): - - if prefix_bundler_name is None: - prefix_bundler_name = "default" - bundler_cls = prefix_bundler_registry[prefix_bundler_name] - return bundler_cls(addon, kernel_name, **kwargs) - diff --git a/jupyterlite_xeus/prefix_bundler/empack_bundler.py b/jupyterlite_xeus/prefix_bundler/empack_bundler.py deleted file mode 100644 index ecd6c81..0000000 --- a/jupyterlite_xeus/prefix_bundler/empack_bundler.py +++ /dev/null @@ -1,48 +0,0 @@ -from pathlib import Path -from empack.pack import DEFAULT_CONFIG_PATH, pack_env -from empack.file_patterns import pkg_file_filter_from_yaml - -from .prefix_bundler_base import PrefixBundlerBase - - - -class EmpackBundler(PrefixBundlerBase): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - - def build(self): - - prefix_path = Path(self.prefix) - - # temp dir for the packed env - out_path = Path(self.cwd.name) / "packed_env" - out_path.mkdir(parents=True, exist_ok=True) - - # Pack the environment (TODO make this configurable) - file_filters = pkg_file_filter_from_yaml(DEFAULT_CONFIG_PATH) - - pack_env( - env_prefix=prefix_path, - relocate_prefix="/", - outdir=out_path, - use_cache=True, - file_filters=file_filters - ) - - # copy all the packages to the packages dir - # (this is shared between multiple xeus-python kernels) - for pkg_path in out_path.iterdir(): - if pkg_path.name.endswith(".tar.gz"): - yield dict( - name=f"xeus:{self.kernel_name}:copy_package:{pkg_path.name}", - actions=[(self.copy_one, [pkg_path, self.packages_dir / pkg_path.name ])], - ) - - # copy the empack_env_meta.json - # this is individual for xeus-python kernel - empack_env_meta = "empack_env_meta.json" - yield dict( - name=f"xeus:{self.kernel_name}:copy_env_file:{empack_env_meta}", - actions=[(self.copy_one, [out_path / empack_env_meta, Path(self.kernel_dir)/ empack_env_meta ])], - ) \ No newline at end of file diff --git a/jupyterlite_xeus/prefix_bundler/noop_prefix_bundler.py b/jupyterlite_xeus/prefix_bundler/noop_prefix_bundler.py deleted file mode 100644 index a71c1ba..0000000 --- a/jupyterlite_xeus/prefix_bundler/noop_prefix_bundler.py +++ /dev/null @@ -1,9 +0,0 @@ -from .prefix_bundler_base import PrefixBundlerBase - -# do nothing at all -class NoopPrefixBundler(PrefixBundlerBase): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def build(self): - yield None \ No newline at end of file diff --git a/jupyterlite_xeus/prefix_bundler/prefix_bundler_base.py b/jupyterlite_xeus/prefix_bundler/prefix_bundler_base.py deleted file mode 100644 index 5b4657d..0000000 --- a/jupyterlite_xeus/prefix_bundler/prefix_bundler_base.py +++ /dev/null @@ -1,19 +0,0 @@ -from pathlib import Path - - - -class PrefixBundlerBase: - # a prefix-bundler takes a prefix (todo, fix arguments / parametrization / api) - # and turns this into something a kernel can consume - def __init__(self, addon, kernel_name): - self.addon = addon - self.cwd = addon.cwd - self.copy_one = addon.copy_one - self.prefix = addon.prefix - self.kernel_name = kernel_name - self.static_dir = addon.static_dir - self.packages_dir = Path(self.static_dir) / "share" / "jupyter" / "kernel_packages" - self.kernel_dir = Path(addon.static_dir) / "share"/ "jupyter"/"kernels"/ kernel_name - - def build(self): - raise NotImplementedError("build method must be implemented by subclass")