Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework xeus output paths and URLs #23

Merged
merged 3 commits into from
Jan 11, 2024
+50 −83
Merged

Rework xeus output paths and URLs #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1039205
Rework xeus output paths and URLs
martinRenou Jan 9, 2024
feb5620
Fix
martinRenou Jan 10, 2024
0379271
Revert to one plugin per-kernel
martinRenou Jan 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 12 additions & 17 deletions jupyterlite_xeus/add_on.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from traitlets import List, Unicode

from .create_conda_env import create_conda_env_from_yaml,create_conda_env_from_specs
from .constants import EXTENSION_NAME, STATIC_DIR
from .constants import EXTENSION_NAME

from empack.pack import DEFAULT_CONFIG_PATH, pack_env, pack_directory, add_tarfile_to_env_meta
from empack.file_patterns import pkg_file_filter_from_yaml
Expand Down Expand Up @@ -74,11 +74,10 @@ class XeusAddon(FederatedExtensionAddon):

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.static_dir = self.output_extensions / STATIC_DIR
self.xeus_output_dir = Path(self.manager.output_dir) / "xeus"
self.cwd = TemporaryDirectory()

def post_build(self, manager):

# check that either prefix or environment_file is set
if not self.prefix and not self.environment_file:
raise ValueError("Either prefix or environment_file must be set")
Expand All @@ -94,7 +93,6 @@ def post_build(self, manager):
yield from self.copy_jupyterlab_extensions_from_prefix(manager)

def create_prefix(self):

# read the environment file
root_prefix = Path(self.cwd.name) / "env"
env_name = "xeus-env"
Expand All @@ -118,13 +116,11 @@ def create_prefix(self):
)

def copy_kernels_from_prefix(self):

if not os.path.exists(self.prefix) or not os.path.isdir(self.prefix):
raise ValueError(f"Prefix {self.prefix} does not exist or is not a directory")

kernel_spec_path = Path(self.prefix) / "share" / "jupyter" / "kernels"


all_kernels = []
# find all folders in the kernelspec path
for kernel_dir in kernel_spec_path.iterdir():
Expand All @@ -139,10 +135,10 @@ def copy_kernels_from_prefix(self):
kernel_file = Path(self.cwd.name) / "kernels.json"
kernel_file.write_text(json.dumps(all_kernels), **UTF8)
yield dict(
name=f"copy:kernels.json",
name=f"copy:{kernel_file}",
actions=[
(
self.copy_one, [kernel_file, self.static_dir / "share"/"jupyter" / "kernels.json" ]
self.copy_one, [kernel_file, self.xeus_output_dir / "kernels.json"]
)
]
)
Expand All @@ -151,38 +147,37 @@ def copy_kernel(self, kernel_dir, kernel_wasm, kernel_js):
kernel_spec = json.loads((kernel_dir / "kernel.json").read_text(**UTF8))

# update kernel_executable path in kernel.json
kernel_spec["argv"][0] = f"bin/{kernel_js.name}"
kernel_spec["argv"][0] = f"xeus/bin/{kernel_js.name}"

# write to temp file
kernel_json = Path(self.cwd.name) / f"{kernel_dir.name}_kernel.json"
kernel_json.write_text(json.dumps(kernel_spec), **UTF8)


# copy the kernel binary files to the bin dir
yield dict(name=f"copy:{kernel_dir.name}:binaries", actions=[
(self.copy_one, [kernel_js, self.static_dir / "bin"/ kernel_js.name ]),
(self.copy_one, [kernel_wasm, self.static_dir / "bin"/ kernel_wasm.name ]),
(self.copy_one, [kernel_js, self.xeus_output_dir / "bin" / kernel_js.name ]),
(self.copy_one, [kernel_wasm, self.xeus_output_dir / "bin" / kernel_wasm.name ]),
])

# copy the kernel.json file
yield dict(
name=f"copy:{kernel_dir.name}:kernel.json",
actions=[(self.copy_one, [kernel_json, self.static_dir /"share"/"jupyter"/ "kernels"/ kernel_dir.name / "kernel.json" ])],
actions=[(self.copy_one, [kernel_json, self.xeus_output_dir / "kernels"/ kernel_dir.name / "kernel.json" ])],
)
# copy the logo files
yield dict(
name=f"copy:{kernel_dir.name}:logos",
actions=[
(self.copy_one, [kernel_dir / "logo-32x32.png", self.static_dir /"share"/ "jupyter"/ "kernels"/ kernel_dir.name / "logo-32x32.png" ]),
(self.copy_one, [kernel_dir / "logo-64x64.png", self.static_dir /"share"/ "jupyter"/ "kernels"/ kernel_dir.name / "logo-64x64.png" ])
(self.copy_one, [kernel_dir / "logo-32x32.png", self.xeus_output_dir / "kernels" / kernel_dir.name / "logo-32x32.png" ]),
(self.copy_one, [kernel_dir / "logo-64x64.png", self.xeus_output_dir / "kernels" / kernel_dir.name / "logo-64x64.png" ])
])

yield from self.pack_prefix(kernel_dir=kernel_dir)

def pack_prefix(self, kernel_dir):
kernel_name = kernel_dir.name
packages_dir = Path(self.static_dir) / "share" / "jupyter" / "kernel_packages"
full_kernel_dir = Path(self.static_dir) / "share"/ "jupyter"/"kernels"/ kernel_name
packages_dir = self.xeus_output_dir / "kernel_packages"
full_kernel_dir = self.xeus_output_dir / "kernels"/ kernel_name

out_path = Path(self.cwd.name) / "packed_env"
out_path.mkdir(parents=True, exist_ok=True)
Expand Down
64 changes: 24 additions & 40 deletions src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Copyright (c) JupyterLite Contributors
// Distributed under the terms of the Modified BSD License.

import { PageConfig, URLExt } from '@jupyterlab/coreutils';
import {
IServiceWorkerManager,
JupyterLiteServer,
Expand All @@ -12,54 +13,25 @@ import { IKernel, IKernelSpecs } from '@jupyterlite/kernel';

import { WebWorkerKernel } from './web_worker_kernel';

const EXTENSION_NAME = 'xeus';
const EXTENSION_STATIC_DIR = `../extensions/@jupyterlite/${EXTENSION_NAME}/static/`;

// helper function to fetch json
function getPkgJson(url: string) {
const json_url = EXTENSION_STATIC_DIR + url;
function getJson(url: string) {
const json_url = URLExt.join(PageConfig.getBaseUrl(), url);
const xhr = new XMLHttpRequest();
xhr.open('GET', json_url, false);
xhr.send(null);
return JSON.parse(xhr.responseText);
}

let kernel_dir: string[] = [];
let kernel_list: string[] = [];
try {
kernel_dir = getPkgJson('share/jupyter/kernels.json');
kernel_list = getJson('xeus/kernels.json');
} catch (err) {
console.log(err);
console.log('could not fetch share/jupyter/kernels/kernels.json');
kernel_dir = [];
console.log(`Could not fetch xeus/kernels.json: ${err}`);
throw err;
}

// fetch kernel spec for each kernel
const kernel_specs = kernel_dir.map(kernel_dir => {
const spec: any = getPkgJson(
'share/jupyter/kernels/' + kernel_dir + '/kernel.json'
);
spec.name = kernel_dir;
spec.dir = kernel_dir;
spec.resources = {
'logo-32x32':
EXTENSION_STATIC_DIR +
'share/jupyter/kernels/' +
kernel_dir +
'/logo-32x32.png',
'logo-64x64':
EXTENSION_STATIC_DIR +
'share/jupyter/kernels/' +
kernel_dir +
'/logo-64x64.png'
};
return spec;
});

const server_kernels = kernel_specs.map(kernelspec => {
const server_kernel: JupyterLiteServerPlugin<void> = {
// use name from spec
id: `@jupyterlite/${kernelspec.name}-extension:kernel`,
const plugins = kernel_list.map((kernel): JupyterLiteServerPlugin<void> => {
return {
id: `@jupyterlite/xeus-${kernel}:register`,
autoStart: true,
requires: [IKernelSpecs],
optional: [IServiceWorkerManager, IBroadcastChannelWrapper],
Expand All @@ -69,6 +41,21 @@ const server_kernels = kernel_specs.map(kernelspec => {
serviceWorker?: IServiceWorkerManager,
broadcastChannel?: IBroadcastChannelWrapper
) => {
// Fetch kernel spec
const kernelspec = getJson('xeus/kernels/' + kernel + '/kernel.json');
kernelspec.name = kernel;
kernelspec.dir = kernel;
kernelspec.resources = {
'logo-32x32': URLExt.join(
PageConfig.getBaseUrl(),
'xeus/kernels/' + kernel + '/logo-32x32.png'
),
'logo-64x64': URLExt.join(
PageConfig.getBaseUrl(),
'xeus/kernels/' + kernel + '/logo-64x64.png'
)
};

kernelspecs.register({
spec: kernelspec,
create: async (options: IKernel.IOptions): Promise<IKernel> => {
Expand All @@ -95,9 +82,6 @@ const server_kernels = kernel_specs.map(kernelspec => {
});
}
};
return server_kernel;
});

const plugins: JupyterLiteServerPlugin<any>[] = server_kernels;

export default plugins;
16 changes: 0 additions & 16 deletions src/web_worker_kernel.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,26 +30,21 @@ export class WebWorkerKernel implements IKernel {
* @param options The instantiation options for a new WebWorkerKernel
*/
constructor(options: WebWorkerKernel.IOptions) {
console.log('constructing WebWorkerKernel kernel');
const { id, name, sendMessage, location } = options;
this._id = id;
this._name = name;
this._location = location;
this._kernelspec = options.kernelspec;
this._sendMessage = sendMessage;
console.log('constructing WebWorkerKernel worker');
this._worker = new Worker(new URL('./worker.js', import.meta.url), {
type: 'module'
});
console.log('constructing WebWorkerKernel done');

this._worker.onmessage = e => {
this._processWorkerMessage(e.data);
};

console.log('wrap');
this._remote = wrap(this._worker);
console.log('wrap done');

this._remote.processMessage({
msg: {
Expand All @@ -60,19 +55,13 @@ export class WebWorkerKernel implements IKernel {
}
});

console.log('init filesystem');
this.initFileSystem(options);

console.log('constructing WebWorkerKernel done2');
}

async handleMessage(msg: KernelMessage.IMessage): Promise<void> {
console.log('handleMessage', msg);
this._parent = msg;
this._parentHeader = msg.header;
console.log('send message to worker');
await this._sendMessageToWorker(msg);
console.log('send message to worker awaiting done');
}

private async _sendMessageToWorker(msg: any): Promise<void> {
Expand All @@ -81,11 +70,7 @@ export class WebWorkerKernel implements IKernel {
this._executeDelegate = new PromiseDelegate<void>();
}

console.log(' this._remote.processMessage({ msg, parent: this.parent });');
await this._remote.processMessage({ msg, parent: this.parent });
console.log(
' this._remote.processMessage({ msg, parent: this.parent }); done'
);
if (msg.header.msg_type !== 'input_reply') {
return await this._executeDelegate.promise;
}
Expand Down Expand Up @@ -120,7 +105,6 @@ export class WebWorkerKernel implements IKernel {
* @param msg The worker message to process.
*/
private _processWorkerMessage(msg: any): void {
console.log('processWorkerMessage', msg);
if (!msg.header) {
return;
}
Expand Down
24 changes: 14 additions & 10 deletions src/worker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
// Copyright (c) JupyterLite Contributors
// Distributed under the terms of the Modified BSD License.

console.log('worker loaded');

import { expose } from 'comlink';

import {
Expand All @@ -12,6 +10,7 @@ import {
IEmscriptenFSNode,
IStats
} from '@jupyterlite/contents';
import { PageConfig, URLExt } from '@jupyterlab/coreutils';

declare function createXeusModule(options: any): any;

Expand Down Expand Up @@ -96,7 +95,6 @@ async function get_stdin() {

class XeusKernel {
constructor(resolve: any) {
console.log('constructing kernel');
this._resolve = resolve;
}

Expand All @@ -105,7 +103,6 @@ class XeusKernel {
}

mount(driveName: string, mountpoint: string, baseUrl: string): void {
console.log('mounting drive');
const { FS, PATH, ERRNO_CODES } = globalThis.Module;

if (!FS) {
Expand Down Expand Up @@ -164,11 +161,14 @@ class XeusKernel {

private async initialize() {
// the location of the kernel on the server
// ie `share/jupyter/kernels/${dir}`
// ie `xeus/kernels/${dir}`
const dir = this._kernelspec.dir;

// location of the kernel binary on the server
const binary_js = this._kernelspec.argv[0];
const binary_js = URLExt.join(
PageConfig.getBaseUrl(),
this._kernelspec.argv[0]
);
const binary_wasm = binary_js.replace('.js', '.wasm');

importScripts(binary_js);
Expand All @@ -182,15 +182,20 @@ class XeusKernel {
});
try {
await this.waitRunDependency();
console.log(globalThis.Module);

// each kernel can have a `async_init` function
// which can do kernel specific **async** initialization
// This function is usually implemented in the pre/post.js
// in the emscripten build of that kernel
if (globalThis.Module['async_init'] !== undefined) {
const kernel_root_url = `share/jupyter/kernels/${dir}`;
const pkg_root_url = 'share/jupyter/kernel_packages';
const kernel_root_url = URLExt.join(
PageConfig.getBaseUrl(),
`xeus/kernels/${dir}`
);
const pkg_root_url = URLExt.join(
PageConfig.getBaseUrl(),
'xeus/kernel_packages'
);
const verbose = true;
await globalThis.Module['async_init'](
kernel_root_url,
Expand Down Expand Up @@ -244,6 +249,5 @@ class XeusKernel {
}

globalThis.ready = new Promise(resolve => {
console.log('expose(new XeusKernel(resolve));');
expose(new XeusKernel(resolve));
});
Loading