Skip to content

Commit

Permalink
Rework xeus output paths and URLs (#23)
Browse files Browse the repository at this point in the history 
* Rework xeus output paths and URLs

* Fix

* Revert to one plugin per-kernel
  • Loading branch information
@martinRenou
martinRenou authored Jan 11, 2024
1 parent 2c3ba57 commit 9a44c1a
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 83 deletions.
29 changes: 12 additions & 17 deletions jupyterlite_xeus/add_on.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from traitlets import List, Unicode

from .create_conda_env import create_conda_env_from_yaml,create_conda_env_from_specs
from .constants import EXTENSION_NAME, STATIC_DIR
from .constants import EXTENSION_NAME

from empack.pack import DEFAULT_CONFIG_PATH, pack_env, pack_directory, add_tarfile_to_env_meta
from empack.file_patterns import pkg_file_filter_from_yaml
Expand Down Expand Up @@ -74,11 +74,10 @@ class XeusAddon(FederatedExtensionAddon):

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.static_dir = self.output_extensions / STATIC_DIR
self.xeus_output_dir = Path(self.manager.output_dir) / "xeus"
self.cwd = TemporaryDirectory()

def post_build(self, manager):

# check that either prefix or environment_file is set
if not self.prefix and not self.environment_file:
raise ValueError("Either prefix or environment_file must be set")
Expand All @@ -94,7 +93,6 @@ def post_build(self, manager):
yield from self.copy_jupyterlab_extensions_from_prefix(manager)

def create_prefix(self):

# read the environment file
root_prefix = Path(self.cwd.name) / "env"
env_name = "xeus-env"
Expand All @@ -118,13 +116,11 @@ def create_prefix(self):
)

def copy_kernels_from_prefix(self):

if not os.path.exists(self.prefix) or not os.path.isdir(self.prefix):
raise ValueError(f"Prefix {self.prefix} does not exist or is not a directory")

kernel_spec_path = Path(self.prefix) / "share" / "jupyter" / "kernels"


all_kernels = []
# find all folders in the kernelspec path
for kernel_dir in kernel_spec_path.iterdir():
Expand All @@ -139,10 +135,10 @@ def copy_kernels_from_prefix(self):
kernel_file = Path(self.cwd.name) / "kernels.json"
kernel_file.write_text(json.dumps(all_kernels), **UTF8)
yield dict(
name=f"copy:kernels.json",
name=f"copy:{kernel_file}",
actions=[
(
self.copy_one, [kernel_file, self.static_dir / "share"/"jupyter" / "kernels.json" ]
self.copy_one, [kernel_file, self.xeus_output_dir / "kernels.json"]
)
]
)
Expand All @@ -151,38 +147,37 @@ def copy_kernel(self, kernel_dir, kernel_wasm, kernel_js):
kernel_spec = json.loads((kernel_dir / "kernel.json").read_text(**UTF8))

# update kernel_executable path in kernel.json
kernel_spec["argv"][0] = f"bin/{kernel_js.name}"
kernel_spec["argv"][0] = f"xeus/bin/{kernel_js.name}"

# write to temp file
kernel_json = Path(self.cwd.name) / f"{kernel_dir.name}_kernel.json"
kernel_json.write_text(json.dumps(kernel_spec), **UTF8)


# copy the kernel binary files to the bin dir
yield dict(name=f"copy:{kernel_dir.name}:binaries", actions=[
(self.copy_one, [kernel_js, self.static_dir / "bin"/ kernel_js.name ]),
(self.copy_one, [kernel_wasm, self.static_dir / "bin"/ kernel_wasm.name ]),
(self.copy_one, [kernel_js, self.xeus_output_dir / "bin" / kernel_js.name ]),
(self.copy_one, [kernel_wasm, self.xeus_output_dir / "bin" / kernel_wasm.name ]),
])

# copy the kernel.json file
yield dict(
name=f"copy:{kernel_dir.name}:kernel.json",
actions=[(self.copy_one, [kernel_json, self.static_dir /"share"/"jupyter"/ "kernels"/ kernel_dir.name / "kernel.json" ])],
actions=[(self.copy_one, [kernel_json, self.xeus_output_dir / "kernels"/ kernel_dir.name / "kernel.json" ])],
)
# copy the logo files
yield dict(
name=f"copy:{kernel_dir.name}:logos",
actions=[
(self.copy_one, [kernel_dir / "logo-32x32.png", self.static_dir /"share"/ "jupyter"/ "kernels"/ kernel_dir.name / "logo-32x32.png" ]),
(self.copy_one, [kernel_dir / "logo-64x64.png", self.static_dir /"share"/ "jupyter"/ "kernels"/ kernel_dir.name / "logo-64x64.png" ])
(self.copy_one, [kernel_dir / "logo-32x32.png", self.xeus_output_dir / "kernels" / kernel_dir.name / "logo-32x32.png" ]),
(self.copy_one, [kernel_dir / "logo-64x64.png", self.xeus_output_dir / "kernels" / kernel_dir.name / "logo-64x64.png" ])
])

yield from self.pack_prefix(kernel_dir=kernel_dir)

def pack_prefix(self, kernel_dir):
kernel_name = kernel_dir.name
packages_dir = Path(self.static_dir) / "share" / "jupyter" / "kernel_packages"
full_kernel_dir = Path(self.static_dir) / "share"/ "jupyter"/"kernels"/ kernel_name
packages_dir = self.xeus_output_dir / "kernel_packages"
full_kernel_dir = self.xeus_output_dir / "kernels"/ kernel_name

out_path = Path(self.cwd.name) / "packed_env"
out_path.mkdir(parents=True, exist_ok=True)
Expand Down
64 changes: 24 additions & 40 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Copyright (c) JupyterLite Contributors
// Distributed under the terms of the Modified BSD License.

import { PageConfig, URLExt } from '@jupyterlab/coreutils';
import {
IServiceWorkerManager,
JupyterLiteServer,
Expand All @@ -12,54 +13,25 @@ import { IKernel, IKernelSpecs } from '@jupyterlite/kernel';

import { WebWorkerKernel } from './web_worker_kernel';

const EXTENSION_NAME = 'xeus';
const EXTENSION_STATIC_DIR = `../extensions/@jupyterlite/${EXTENSION_NAME}/static/`;

// helper function to fetch json
function getPkgJson(url: string) {
const json_url = EXTENSION_STATIC_DIR + url;
function getJson(url: string) {
const json_url = URLExt.join(PageConfig.getBaseUrl(), url);
const xhr = new XMLHttpRequest();
xhr.open('GET', json_url, false);
xhr.send(null);
return JSON.parse(xhr.responseText);
}

let kernel_dir: string[] = [];
let kernel_list: string[] = [];
try {
kernel_dir = getPkgJson('share/jupyter/kernels.json');
kernel_list = getJson('xeus/kernels.json');
} catch (err) {
console.log(err);
console.log('could not fetch share/jupyter/kernels/kernels.json');
kernel_dir = [];
console.log(`Could not fetch xeus/kernels.json: ${err}`);
throw err;
}

// fetch kernel spec for each kernel
const kernel_specs = kernel_dir.map(kernel_dir => {
const spec: any = getPkgJson(
'share/jupyter/kernels/' + kernel_dir + '/kernel.json'
);
spec.name = kernel_dir;
spec.dir = kernel_dir;
spec.resources = {
'logo-32x32':
EXTENSION_STATIC_DIR +
'share/jupyter/kernels/' +
kernel_dir +
'/logo-32x32.png',
'logo-64x64':
EXTENSION_STATIC_DIR +
'share/jupyter/kernels/' +
kernel_dir +
'/logo-64x64.png'
};
return spec;
});

const server_kernels = kernel_specs.map(kernelspec => {
const server_kernel: JupyterLiteServerPlugin<void> = {
// use name from spec
id: `@jupyterlite/${kernelspec.name}-extension:kernel`,
const plugins = kernel_list.map((kernel): JupyterLiteServerPlugin<void> => {
return {
id: `@jupyterlite/xeus-${kernel}:register`,
autoStart: true,
requires: [IKernelSpecs],
optional: [IServiceWorkerManager, IBroadcastChannelWrapper],
Expand All @@ -69,6 +41,21 @@ const server_kernels = kernel_specs.map(kernelspec => {
serviceWorker?: IServiceWorkerManager,
broadcastChannel?: IBroadcastChannelWrapper
) => {
// Fetch kernel spec
const kernelspec = getJson('xeus/kernels/' + kernel + '/kernel.json');
kernelspec.name = kernel;
kernelspec.dir = kernel;
kernelspec.resources = {
'logo-32x32': URLExt.join(
PageConfig.getBaseUrl(),
'xeus/kernels/' + kernel + '/logo-32x32.png'
),
'logo-64x64': URLExt.join(
PageConfig.getBaseUrl(),
'xeus/kernels/' + kernel + '/logo-64x64.png'
)
};

kernelspecs.register({
spec: kernelspec,
create: async (options: IKernel.IOptions): Promise<IKernel> => {
Expand All @@ -95,9 +82,6 @@ const server_kernels = kernel_specs.map(kernelspec => {
});
}
};
return server_kernel;
});

const plugins: JupyterLiteServerPlugin<any>[] = server_kernels;

export default plugins;
16 changes: 0 additions & 16 deletions src/web_worker_kernel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,26 +30,21 @@ export class WebWorkerKernel implements IKernel {
* @param options The instantiation options for a new WebWorkerKernel
*/
constructor(options: WebWorkerKernel.IOptions) {
console.log('constructing WebWorkerKernel kernel');
const { id, name, sendMessage, location } = options;
this._id = id;
this._name = name;
this._location = location;
this._kernelspec = options.kernelspec;
this._sendMessage = sendMessage;
console.log('constructing WebWorkerKernel worker');
this._worker = new Worker(new URL('./worker.js', import.meta.url), {
type: 'module'
});
console.log('constructing WebWorkerKernel done');

this._worker.onmessage = e => {
this._processWorkerMessage(e.data);
};

console.log('wrap');
this._remote = wrap(this._worker);
console.log('wrap done');

this._remote.processMessage({
msg: {
Expand All @@ -60,19 +55,13 @@ export class WebWorkerKernel implements IKernel {
}
});

console.log('init filesystem');
this.initFileSystem(options);

console.log('constructing WebWorkerKernel done2');
}

async handleMessage(msg: KernelMessage.IMessage): Promise<void> {
console.log('handleMessage', msg);
this._parent = msg;
this._parentHeader = msg.header;
console.log('send message to worker');
await this._sendMessageToWorker(msg);
console.log('send message to worker awaiting done');
}

private async _sendMessageToWorker(msg: any): Promise<void> {
Expand All @@ -81,11 +70,7 @@ export class WebWorkerKernel implements IKernel {
this._executeDelegate = new PromiseDelegate<void>();
}

console.log(' this._remote.processMessage({ msg, parent: this.parent });');
await this._remote.processMessage({ msg, parent: this.parent });
console.log(
' this._remote.processMessage({ msg, parent: this.parent }); done'
);
if (msg.header.msg_type !== 'input_reply') {
return await this._executeDelegate.promise;
}
Expand Down Expand Up @@ -120,7 +105,6 @@ export class WebWorkerKernel implements IKernel {
* @param msg The worker message to process.
*/
private _processWorkerMessage(msg: any): void {
console.log('processWorkerMessage', msg);
if (!msg.header) {
return;
}
Expand Down
24 changes: 14 additions & 10 deletions src/worker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
// Copyright (c) JupyterLite Contributors
// Distributed under the terms of the Modified BSD License.

console.log('worker loaded');

import { expose } from 'comlink';

import {
Expand All @@ -12,6 +10,7 @@ import {
IEmscriptenFSNode,
IStats
} from '@jupyterlite/contents';
import { PageConfig, URLExt } from '@jupyterlab/coreutils';

declare function createXeusModule(options: any): any;

Expand Down Expand Up @@ -96,7 +95,6 @@ async function get_stdin() {

class XeusKernel {
constructor(resolve: any) {
console.log('constructing kernel');
this._resolve = resolve;
}

Expand All @@ -105,7 +103,6 @@ class XeusKernel {
}

mount(driveName: string, mountpoint: string, baseUrl: string): void {
console.log('mounting drive');
const { FS, PATH, ERRNO_CODES } = globalThis.Module;

if (!FS) {
Expand Down Expand Up @@ -164,11 +161,14 @@ class XeusKernel {

private async initialize() {
// the location of the kernel on the server
// ie `share/jupyter/kernels/${dir}`
// ie `xeus/kernels/${dir}`
const dir = this._kernelspec.dir;

// location of the kernel binary on the server
const binary_js = this._kernelspec.argv[0];
const binary_js = URLExt.join(
PageConfig.getBaseUrl(),
this._kernelspec.argv[0]
);
const binary_wasm = binary_js.replace('.js', '.wasm');

importScripts(binary_js);
Expand All @@ -182,15 +182,20 @@ class XeusKernel {
});
try {
await this.waitRunDependency();
console.log(globalThis.Module);

// each kernel can have a `async_init` function
// which can do kernel specific **async** initialization
// This function is usually implemented in the pre/post.js
// in the emscripten build of that kernel
if (globalThis.Module['async_init'] !== undefined) {
const kernel_root_url = `share/jupyter/kernels/${dir}`;
const pkg_root_url = 'share/jupyter/kernel_packages';
const kernel_root_url = URLExt.join(
PageConfig.getBaseUrl(),
`xeus/kernels/${dir}`
);
const pkg_root_url = URLExt.join(
PageConfig.getBaseUrl(),
'xeus/kernel_packages'
);
const verbose = true;
await globalThis.Module['async_init'](
kernel_root_url,
Expand Down Expand Up @@ -244,6 +249,5 @@ class XeusKernel {
}

globalThis.ready = new Promise(resolve => {
console.log('expose(new XeusKernel(resolve));');
expose(new XeusKernel(resolve));
});

0 comments on commit 9a44c1a

Please sign in to comment.