diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1c24875..3022fd6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,9 +36,6 @@ jobs: run: | npm install - - name: Fetch wasm dependencies - run: npm run fetch:wasm - - name: Build run: | npm run build diff --git a/.gitignore b/.gitignore index 30d7d3d..ce7734c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,8 @@ test/package-lock.json *.js *.wasm cockle_wasm_env/ +cockle-config.json demo/package-lock.json +demo/assets/*.json +test/assets/*.json src/version.ts diff --git a/README.md b/README.md index 35f2337..f3edcc9 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ a `micromamba` environment as part of the `npm prepack` process. micromamba env create -f environment-dev.yml -y micromamba activate cockle npm install -npm run fetch:wasm npm run build npm run lint:check ``` diff --git a/cockle-config-base.json b/cockle-config-base.json new file mode 100644 index 0000000..1e7a0bb --- /dev/null +++ b/cockle-config-base.json @@ -0,0 +1,23 @@ +[ + { + "package": "cockle_fs", + "modules": [ + { + "name": "fs", + "commands": "" + } + ] + }, + { + "package": "coreutils", + "modules": [ + { + "name": "coreutils", + "commands": "basename,cat,chmod,cp,cut,date,dir,dircolors,dirname,echo,env,expr,head,id,join,ln,logname,ls,md5sum,mkdir,mv,nl,pwd,realpath,rm,rmdir,seq,sha1sum,sha224sum,sha256sum,sha384sum,sha512sum,sleep,sort,stat,stty,tail,touch,tr,tty,uname,uniq,vdir,wc" + } + ] + }, + { + "package": "grep" + } +] diff --git a/demo/cockle-config-in.json b/demo/cockle-config-in.json new file mode 100644 index 0000000..f77280a --- /dev/null +++ b/demo/cockle-config-in.json @@ -0,0 +1,5 @@ +[ + { + "package": "lua" + } +] diff --git a/demo/package.json b/demo/package.json index ded608c..28a34e7 100644 --- a/demo/package.json +++ b/demo/package.json @@ -8,8 +8,8 @@ "types": "lib/index.d.ts", "private": true, "scripts": { - "build": "npm run build:wasm && rspack build", - "build:wasm": "cp node_modules/@jupyterlite/cockle/lib/wasm/*.js assets/ && cp node_modules/@jupyterlite/cockle/lib/wasm/*.wasm assets/", + "build": "rspack build", + "postbuild": "node node_modules/@jupyterlite/cockle/lib/tools/prepare_wasm.js --copy assets", "serve": "rspack serve" }, "devDependencies": { diff --git a/package.json b/package.json index c60c7f3..658cf8b 100644 --- a/package.json +++ b/package.json @@ -14,22 +14,20 @@ }, "files": [ "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,wasm,woff2,ttf}", - "src/**/*.ts" + "src/**/*.ts", + "cockle-config-base.json" ], "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "fetch:wasm:create-env": "micromamba create -p $(pwd)/cockle_wasm_env -y cockle_fs grep coreutils lua --platform=emscripten-wasm32 -c https://repo.mamba.pm/emscripten-forge -c https://repo.mamba.pm/conda-forge", - "fetch:wasm:copy": "mkdir -p src/wasm && cp $(pwd)/cockle_wasm_env/bin/*.js src/wasm/ && cp $(pwd)/cockle_wasm_env/bin/*.wasm src/wasm/", - "fetch:wasm": "npm run fetch:wasm:create-env && npm run fetch:wasm:copy", - "build": "tsc && npm run build:worker && cp src/wasm/*.wasm lib/wasm/", + "build": "tsc && npm run build:worker", "build:worker": "rspack --config worker.rspack.config.js --mode=development", "eslint": "npm run eslint:check -- --fix", "eslint:check": "eslint . --cache --ext .ts,.tsx", "lint": "npm run prettier && npm run eslint", "lint:check": "npm run prettier:check && npm run eslint:check", "prebuild": "node -p \"'export const COCKLE_VERSION = ' + JSON.stringify(require('./package.json').version) + ';'\" > src/version.ts", - "prepack": "npm install && npm run fetch:wasm && npm run build", + "prepack": "npm install && npm run build", "prettier": "prettier --list-different --write \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md,.yml}\"", "prettier:check": "prettier --list-different \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md,.yml}\"" }, diff --git a/src/command_registry.ts b/src/command_registry.ts index bf3e505..6d7c46f 100644 --- a/src/command_registry.ts +++ b/src/command_registry.ts @@ -1,27 +1,10 @@ import { ICommandRunner } from './commands/command_runner'; -import { CoreutilsCommandRunner } from './commands/coreutils_command_runner'; -import { GrepCommandRunner } from './commands/grep_command_runner'; -import { LuaCommandRunner } from './commands/lua_command_runner'; import * as AllBuiltinCommands from './builtin'; import { WasmLoader } from './wasm_loader'; export class CommandRegistry { constructor(wasmLoader: WasmLoader) { this.registerBuiltinCommands(AllBuiltinCommands); - - this._commandRunners = [ - new CoreutilsCommandRunner(wasmLoader), - new GrepCommandRunner(wasmLoader), - new LuaCommandRunner(wasmLoader) - ]; - - // Command name -> runner mapping - // Should probably check not overwriting any command names - for (const runner of this._commandRunners) { - for (const name of runner.names()) { - this._map.set(name, runner); - } - } } get(name: string): ICommandRunner | null { @@ -33,10 +16,13 @@ export class CommandRegistry { } /** - * Register a command runner under a single name. + * Register a command runner under all of its names. */ - register(name: string, commandRunner: ICommandRunner) { - this._map.set(name, commandRunner); + register(commandRunner: ICommandRunner) { + // Should probably check not overwriting any command names + for (const name of commandRunner.names()) { + this._map.set(name, commandRunner); + } } registerBuiltinCommands(commands: any) { @@ -47,7 +33,7 @@ export class CommandRegistry { try { const obj = new (cls as any)(); if (obj instanceof AllBuiltinCommands.BuiltinCommand) { - this.register(obj.name, obj); + this.register(obj); } } catch { // If there is any problem registering a command runner this way, silently fail. @@ -55,6 +41,5 @@ export class CommandRegistry { } } - private _commandRunners: ICommandRunner[]; private _map: Map = new Map(); } diff --git a/src/commands/coreutils_command_runner.ts b/src/commands/coreutils_command_runner.ts deleted file mode 100644 index 05e5713..0000000 --- a/src/commands/coreutils_command_runner.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { WasmCommandRunner } from './wasm_command_runner'; -import { WasmLoader } from '../wasm_loader'; - -export class CoreutilsCommandRunner extends WasmCommandRunner { - constructor(wasmLoader: WasmLoader) { - super(wasmLoader); - } - - moduleName(): string { - return 'coreutils'; - } - - names(): string[] { - return [ - 'basename', - 'cat', - 'chmod', - 'cp', - 'cut', - 'date', - 'dir', - 'dircolors', - 'dirname', - 'echo', - 'env', - 'expr', - 'head', - 'id', - 'join', - 'ln', - 'logname', - 'ls', - 'md5sum', - 'mkdir', - 'mv', - 'nl', - 'pwd', - 'realpath', - 'rm', - 'rmdir', - 'seq', - 'sha1sum', - 'sha224sum', - 'sha256sum', - 'sha384sum', - 'sha512sum', - 'sleep', - 'sort', - 'stat', - 'stty', - 'tail', - 'touch', - 'tr', - 'tty', - 'uname', - 'uniq', - 'vdir', - 'wc' - ]; - } -} diff --git a/src/commands/grep_command_runner.ts b/src/commands/grep_command_runner.ts deleted file mode 100644 index 6c2657d..0000000 --- a/src/commands/grep_command_runner.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { WasmCommandRunner } from './wasm_command_runner'; -import { WasmLoader } from '../wasm_loader'; - -export class GrepCommandRunner extends WasmCommandRunner { - constructor(wasmLoader: WasmLoader) { - super(wasmLoader); - } - - moduleName(): string { - return 'grep'; - } - - names(): string[] { - return ['grep']; - } -} diff --git a/src/commands/lua_command_runner.ts b/src/commands/lua_command_runner.ts deleted file mode 100644 index 9f597ca..0000000 --- a/src/commands/lua_command_runner.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { WasmCommandRunner } from './wasm_command_runner'; -import { WasmLoader } from '../wasm_loader'; - -export class LuaCommandRunner extends WasmCommandRunner { - constructor(wasmLoader: WasmLoader) { - super(wasmLoader); - } - - moduleName(): string { - return 'lua'; - } - - names(): string[] { - return ['lua']; - } -} diff --git a/src/commands/wasm_command_runner.ts b/src/commands/wasm_command_runner.ts index 5e113dc..f116fe8 100644 --- a/src/commands/wasm_command_runner.ts +++ b/src/commands/wasm_command_runner.ts @@ -3,12 +3,20 @@ import { Context } from '../context'; import { ExitCode } from '../exit_code'; import { WasmLoader } from '../wasm_loader'; -export abstract class WasmCommandRunner implements ICommandRunner { - constructor(readonly wasmLoader: WasmLoader) {} - - abstract moduleName(): string; +export class WasmCommandRunner implements ICommandRunner { + constructor( + readonly wasmLoader: WasmLoader, + readonly _moduleName: string, + readonly _commandNames: string[] + ) {} + + moduleName(): string { + return this._moduleName; + } - abstract names(): string[]; + names(): string[] { + return this._commandNames; + } async run(cmdName: string, context: Context): Promise { const { args, fileSystem, mountpoint, stdin, stdout, stderr } = context; diff --git a/src/shell_impl.ts b/src/shell_impl.ts index cd89499..35a3c88 100644 --- a/src/shell_impl.ts +++ b/src/shell_impl.ts @@ -14,6 +14,7 @@ import { FileInput, FileOutput, IInput, IOutput, Pipe, TerminalInput, TerminalOu import { CommandNode, PipeNode, parse } from './parse'; import { longestStartsWith, toColumns } from './utils'; import { WasmLoader } from './wasm_loader'; +import { WasmCommandRunner } from './commands/wasm_command_runner'; /** * Shell implementation. @@ -38,6 +39,7 @@ export class ShellImpl implements IShell { } async initialize() { + await this._initWasmPackages(); await this._initFilesystem(); } @@ -219,6 +221,38 @@ export class ShellImpl implements IShell { } } + private async _initWasmPackages(): Promise { + const url = this.options.wasmBaseUrl + 'cockle-config.json'; + const response = await fetch(url); + if (!response.ok) { + // Would be nice to report this via the terminal. + console.error(`Failed to fetch ${url}, terminal cannot function without it`); + } + + const cockleConfig = await response.json(); + // Check JSON follows schema? + // May want to store JSON config. + + const packageNames = cockleConfig.map((x: any) => x.package); + const fsPackage = 'cockle_fs'; + if (!packageNames.includes(fsPackage)) { + console.error(`cockle-config.json does not include required package '${fsPackage}'`); + } + + // Create a command runners for each wasm module of each emscripten-forge package. + for (const pkgConfig of cockleConfig) { + if (pkgConfig.package === fsPackage) { + continue; + } + + for (const module of pkgConfig.modules) { + const commands = module.commands.split(','); + const runner = new WasmCommandRunner(this._wasmLoader, module.name, commands); + this._commandRegistry.register(runner); + } + } + } + private async _runCommands(cmdText: string): Promise { this.options.enableBufferedStdinCallback(true); diff --git a/src/tools/prepare_wasm.ts b/src/tools/prepare_wasm.ts new file mode 100644 index 0000000..b4eeec0 --- /dev/null +++ b/src/tools/prepare_wasm.ts @@ -0,0 +1,116 @@ +/** + * Prepare wasm packages from emscripten-forge so that they are available at runtime. + * Uses ../../cockle-config-base.json relative to the directory of this script and optional + * cockle-config-in.json in pwd to determine which packages are required. + * Creates a micromamba environment containing the wasm packages and either copies the files + * to the specified statically-served assets directory or writes a file containing the names + * of the files to be so copied, depending on the arguments passed to this script. + */ + +/* eslint-disable */ +const fs = require('node:fs'); +const path = require('node:path'); +const { execSync } = require('node:child_process'); +/* eslint-enable */ + +const ENV_NAME = 'cockle_wasm_env'; +const MICROMAMBA_COMMAND = 'micromamba'; +const PLATFORM = 'emscripten-wasm32'; +const REPOS = '-c https://repo.mamba.pm/emscripten-forge -c https://repo.mamba.pm/conda-forge'; + +if (process.argv.length !== 4 || (process.argv[2] !== '--list' && process.argv[2] !== '--copy')) { + console.log('Usage: prepare_wasm --list list-filename'); + console.log('Usage: prepare_wasm --copy target-directory'); + process.exit(1); +} +const wantCopy = process.argv[2] === '--copy'; +const target = process.argv[3]; + +// Base cockle config file from this repo. +const baseConfigFilename = path.join(__dirname, '..', '..', 'cockle-config-base.json'); +console.log('Using base config', baseConfigFilename); +let cockleConfig = JSON.parse(fs.readFileSync(baseConfigFilename, 'utf-8')); + +// Optional extra cockle config file from pwd. +const otherConfigFilename = path.join(process.cwd(), 'cockle-config-in.json'); +if (fs.existsSync(otherConfigFilename)) { + console.log('Combining with config from', otherConfigFilename); + const extraConfig = JSON.parse(fs.readFileSync(otherConfigFilename, 'utf-8')); + cockleConfig = cockleConfig.concat(extraConfig); +} + +// Required emscripten-wasm32 packages. +const packageNames = cockleConfig.map((item: any) => item.package); +console.log('Required package names', packageNames); + +// Create or reuse existing mamba environment for the wasm packages. +const envPath = `./${ENV_NAME}`; +if (fs.existsSync(envPath)) { + console.log(`Using existing environment in ${envPath}`); + // Should really check that env contents are what we want. +} else { + const suffix = `--platform=${PLATFORM} ${REPOS}`; + console.log(`Creating new environment in ${envPath}`); + const createEnvCmd = `${MICROMAMBA_COMMAND} create -p ${envPath} -y ${packageNames.join(' ')} ${suffix}`; + console.log(execSync(createEnvCmd).toString()); +} + +// Obtain wasm package info such as version and build string. +const wasmPackageInfo = JSON.parse( + execSync(`${MICROMAMBA_COMMAND} run -p ${envPath} ${MICROMAMBA_COMMAND} list --json`).toString() +); +//console.log('Wasm package info:', wasmPackageInfo); + +// Insert package info into cockle config. +for (const packageConfig of cockleConfig) { + const packageName = packageConfig.package; + const info = wasmPackageInfo.find((x: any) => x.name === packageName); + if (info === undefined) { + throw Error(`Do not have package info for ${packageName}`); + } + + console.log(`Add package info to ${packageName}`); + for (const prop of ['build_string', 'platform', 'version']) { + packageConfig[prop] = info[prop]; + } + + // Fill in defaults. + if (!Object.hasOwn(packageConfig, 'modules')) { + console.log(`Adding default module for ${packageName}`); + packageConfig.modules = [{ name: packageName }]; + } + for (const module of packageConfig.modules) { + if (!Object.hasOwn(module, 'commands')) { + console.log(`Adding default commands for ${packageName} module ${module.name}`); + module.commands = module.name; + } + } +} + +// Output config file. +let targetConfigFile = 'cockle-config.json'; +if (wantCopy) { + targetConfigFile = path.join(target, targetConfigFile); +} +fs.writeFileSync(targetConfigFile, JSON.stringify(cockleConfig, null, 2)); +const filenames = [targetConfigFile]; + +// Output wasm files and their javascript wrappers. +const moduleNames = cockleConfig.flatMap((x: any) => x.modules).map((x: any) => x.name); +for (const moduleName of moduleNames) { + for (const suffix of ['.js', '.wasm']) { + const filename = moduleName + suffix; + const srcFilename = path.join(envPath, 'bin', filename); + if (wantCopy) { + const targetFileName = path.join(target, filename); + fs.copyFileSync(srcFilename, targetFileName); + } else { + filenames.push(path.join(envPath, 'bin', moduleName + suffix)); + } + } +} + +if (!wantCopy) { + console.log('Writing list of required files'); + fs.writeFileSync(target, filenames.join('\n')); +} diff --git a/test/cockle-config-in.json b/test/cockle-config-in.json new file mode 100644 index 0000000..f77280a --- /dev/null +++ b/test/cockle-config-in.json @@ -0,0 +1,5 @@ +[ + { + "package": "lua" + } +] diff --git a/test/package.json b/test/package.json index 124640b..035e203 100644 --- a/test/package.json +++ b/test/package.json @@ -8,8 +8,8 @@ "types": "lib/index.d.ts", "private": true, "scripts": { - "build": "npm run build:wasm && rspack build", - "build:wasm": "cp node_modules/@jupyterlite/cockle/lib/wasm/*.js assets/ && cp node_modules/@jupyterlite/cockle/lib/wasm/*.wasm assets/", + "build": "rspack build", + "postbuild": "node node_modules/@jupyterlite/cockle/lib/tools/prepare_wasm.js --copy assets", "serve": "rspack serve", "test": "playwright test", "test:ui": "playwright test --ui", diff --git a/test/serve/shell_setup.ts b/test/serve/shell_setup.ts index 920e1dd..bcd24d0 100644 --- a/test/serve/shell_setup.ts +++ b/test/serve/shell_setup.ts @@ -50,5 +50,10 @@ async function _shell_setup_common(options: IOptions, level: number): Promise new Promise(resolve => setTimeout(resolve, ms)); + await sleep(20); + return { shell, output }; }