From 9b42bbd791d8e21608adaaee3cb411a0debb4420 Mon Sep 17 00:00:00 2001 From: Muspi Merol Date: Fri, 23 Aug 2024 20:27:08 +0800 Subject: [PATCH] feat: modularization --- reasonify-headless/api/fs.py | 51 ++++++++++++++++++++++++++++++ reasonify-headless/bridge.pyi | 5 +++ reasonify-headless/load.py | 58 +--------------------------------- reasonify-headless/utils/fs.py | 19 +++++++++++ src/lib/components/Chat.svelte | 2 ++ src/lib/py/api.ts | 17 ++++++++++ src/lib/py/fs.ts | 6 ++-- src/lib/py/load.ts | 12 ++++--- 8 files changed, 105 insertions(+), 65 deletions(-) create mode 100644 reasonify-headless/api/fs.py create mode 100644 reasonify-headless/bridge.pyi create mode 100644 reasonify-headless/utils/fs.py create mode 100644 src/lib/py/api.ts diff --git a/reasonify-headless/api/fs.py b/reasonify-headless/api/fs.py new file mode 100644 index 0000000..4e6e005 --- /dev/null +++ b/reasonify-headless/api/fs.py @@ -0,0 +1,51 @@ +import sys +from functools import cache +from os import chdir +from pathlib import Path + +from bridge import with_toast +from js import window +from pyodide.ffi import create_once_callable, create_proxy +from utils.fs import NativeFS, mount + + +@cache +def _install_slugify(): + @with_toast("installing slugify") + @create_once_callable + async def install_slugify(): + from reasonify.tools.install import pip_install + + await pip_install("python-slugify") + + return install_slugify() + + +mounted: dict[str, NativeFS] = {} + + +@create_proxy +async def mount_native_fs(): + install_promise = _install_slugify() + + handle = await window.showDirectoryPicker() + while await handle.requestPermission({"mode": "readwrite"}) != "granted": + pass + + await install_promise + + from slugify import slugify + + name = slugify(handle.name) + + fs = await mount(path := str(root / name), handle) + + mounted[path] = fs + + return name + + +root = Path("/workspace/mnt") +root.mkdir(parents=True, exist_ok=True) +chdir(root) +sys.path.append(str(root)) diff --git a/reasonify-headless/bridge.pyi b/reasonify-headless/bridge.pyi new file mode 100644 index 0000000..47451b1 --- /dev/null +++ b/reasonify-headless/bridge.pyi @@ -0,0 +1,5 @@ +from typing import Callable + +from pyodide.webloop import PyodideFuture + +def with_toast[**Params, Return](message: str) -> Callable[[Callable[Params, Return]], Callable[Params, PyodideFuture[Return]]]: ... diff --git a/reasonify-headless/load.py b/reasonify-headless/load.py index f2f71f9..cc3806b 100644 --- a/reasonify-headless/load.py +++ b/reasonify-headless/load.py @@ -1,19 +1,10 @@ import sys -from functools import cache -from os import chdir from pathlib import Path -from typing import TYPE_CHECKING, Awaitable, Callable - -from js import FileSystemDirectoryHandle, window -from micropip import install -from pyodide.ffi import create_once_callable, create_proxy -from pyodide.webloop import PyodideFuture +from typing import TYPE_CHECKING if TYPE_CHECKING: sources: dict[str, str] = {} - def with_toast[**Params, Return](message: str) -> Callable[[Callable[Params, Return]], Callable[Params, PyodideFuture[Return]]]: ... - for path, source in sources.items(): file = Path("/home/pyodide", path) @@ -26,50 +17,3 @@ def with_toast[**Params, Return](message: str) -> Callable[[Callable[Params, Ret for module in tuple(sys.modules): # for HMR if module.startswith("reasonify"): sys.modules.pop(module).__dict__.clear() - -if TYPE_CHECKING: - - class NativeFS: - syncfs: Callable[[], Awaitable[None]] - - def mount(target: str, handle: FileSystemDirectoryHandle) -> Awaitable[NativeFS]: ... - -else: - from pyodide_js import mountNativeFS as mount - - -@cache -def install_slugify(): - @with_toast("installing slugify") - @create_once_callable - async def install_slugify(): - await install("python-slugify") - - return install_slugify() - - -@create_proxy -async def mount_native_fs(): - handle = await window.showDirectoryPicker() - while await handle.requestPermission({"mode": "readwrite"}) != "granted": - pass - - await install_slugify() - from slugify import slugify - - name = slugify(handle.name) - - fs = await mount(path := str(root / name), handle) - - mounted[path] = fs - - return name - - -root = Path("/workspace/mnt") - -root.mkdir(parents=True, exist_ok=True) - -chdir(root) - -mounted: dict[str, "NativeFS"] = {} diff --git a/reasonify-headless/utils/fs.py b/reasonify-headless/utils/fs.py new file mode 100644 index 0000000..35e3983 --- /dev/null +++ b/reasonify-headless/utils/fs.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING, Awaitable, Callable + +from js import FileSystemDirectoryHandle + +if TYPE_CHECKING: + + class NativeFS: + syncfs: Callable[[], Awaitable[None]] + + def mount(target: str, handle: FileSystemDirectoryHandle) -> Awaitable[NativeFS]: ... + def unmount(target: str) -> None: ... + +else: + from pyodide_js import mountNativeFS as mount + from pyodide_js.FS import unmount + + NativeFS = object + +__all__ = ["mount", "unmount", "NativeFS"] diff --git a/src/lib/components/Chat.svelte b/src/lib/components/Chat.svelte index 79f5049..4f375a8 100644 --- a/src/lib/components/Chat.svelte +++ b/src/lib/components/Chat.svelte @@ -3,6 +3,7 @@ import type { PythonError } from "pyodide/ffi"; import { type Chain, initChain } from "../py"; + import { clearApiCache } from "../py/api"; import Highlight from "./Highlight.svelte"; import Markdown from "./Markdown.svelte"; import { dev } from "$app/environment"; @@ -23,6 +24,7 @@ refreshing = true; try { $reasonifyReady = false; + clearApiCache(); chain = await initChain(); } finally { refreshing = false; diff --git a/src/lib/py/api.ts b/src/lib/py/api.ts new file mode 100644 index 0000000..ebd0a64 --- /dev/null +++ b/src/lib/py/api.ts @@ -0,0 +1,17 @@ +import { py } from "./load"; + +const cache = new Map(); + +export function getApi(slug: string): T { + if (cache.has(slug)) { + return cache.get(slug) as T; + } else { + const api = py.pyimport(slug) as T; + cache.set(slug, api); + return api; + } +} + +export function clearApiCache() { + cache.clear(); +} diff --git a/src/lib/py/fs.ts b/src/lib/py/fs.ts index 5420948..9c718be 100644 --- a/src/lib/py/fs.ts +++ b/src/lib/py/fs.ts @@ -1,9 +1,7 @@ -import getGlobals from "./globals"; +import { getApi } from "./api"; import { withToast } from "$lib/utils/toast"; -const getMount = getGlobals<() => Promise>("mount_native_fs"); - export async function mount(): Promise { - const mount = await getMount(); + const mount = getApi<() => Promise>("api.fs.mount_native_fs"); return withToast("selecting a directory to mount", (name) => (`mounted ${name}`))(mount)(); } diff --git a/src/lib/py/load.ts b/src/lib/py/load.ts index e41b2d4..eab071b 100644 --- a/src/lib/py/load.ts +++ b/src/lib/py/load.ts @@ -3,14 +3,18 @@ import { dev } from "$app/environment"; import * as env from "$env/static/public"; import { cacheSingleton } from "$lib/utils/cache"; import { withToast } from "$lib/utils/toast"; -import { version } from "pyodide"; +import { type PyodideInterface, version } from "pyodide"; const indexURL = ("PUBLIC_PYODIDE_INDEX_URL" in env) ? (env.PUBLIC_PYODIDE_INDEX_URL as string).replace("{}", version) : `https://cdn.jsdelivr.net/pyodide/v${version}/full/`; export const getPy = cacheSingleton(async () => { const { loadPyodide } = await import("pyodide"); - const py = await loadPyodide({ indexURL, packages: ["micropip", "typing-extensions"], args: dev ? [] : ["-O"] }); + const pyodide = await loadPyodide({ indexURL, packages: ["micropip", "typing-extensions"], args: dev ? [] : ["-O"] }); + pyodide.registerJsModule("bridge", { with_toast: withToast }); pyodideReady.set(true); - py.globals.set("with_toast", withToast); - return py; + py = pyodide; + return pyodide; }); + +// eslint-disable-next-line import/no-mutable-exports +export let py: PyodideInterface;