-
-
Notifications
You must be signed in to change notification settings - Fork 50
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #171 from RicBent/wasm
Added JavaScript/wasm target via emscripten
- Loading branch information
Showing
30 changed files
with
3,190 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| add_executable( "${PROJECT_NAME}-wasm" | ||
| kiwi_wasm.cpp | ||
| ) | ||
|
|
||
| target_link_libraries( "${PROJECT_NAME}-wasm" | ||
| "${PROJECT_NAME}_static" | ||
| ) | ||
|
|
||
| set_target_properties("${PROJECT_NAME}-wasm" PROPERTIES | ||
| LINK_FLAGS "--bind -s WASM=1 -s ALLOW_MEMORY_GROWTH=1 -s EXPORT_ES6=1 -s MODULARIZE=1 -s EXPORT_NAME=kiwi -s 'EXPORTED_RUNTIME_METHODS=[\"FS\"]'" | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| # kiwi-nlp, 한국어 형태소 분석기 Kiwi의 TypeScript/JavaScript 바인딩 | ||
|
|
||
| ## Building | ||
|
|
||
| Additionally to the requirements of the main project, you need to install [Emscripten](https://emscripten.org/docs/getting_started/downloads.html) and [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). | ||
|
|
||
| To build the package, simply run `./build.sh`. | ||
|
|
||
| This is currently only supported on Linux and macOS. You can run the build script on Windows by using [WSL](https://learn.microsoft.com/en-us/windows/wsl/install). | ||
|
|
||
| You can pass the `--demo` flag to build the demo in `package-demo` as well. | ||
| If you pass `--demo-dev`, a development server for the demo will be started. | ||
|
|
||
| Running the above command also automatically upgrades to package version if it doesn't match the version in the main project. | ||
|
|
||
| ## Documentation | ||
|
|
||
| The documentation for the package can be generated by running `npm run doc` inside the `package` directory. | ||
|
|
||
| The main entry point for the API is `KiwiBuilder`, which is used to create instances `Kiwi`. | ||
|
|
||
| ## Example Usage | ||
|
|
||
| ```javascript | ||
| import { KiwiBuilder, Match } from 'kiwi-nlp'; | ||
|
|
||
| async function example() { | ||
| const builder = await KiwiBuilder.create('path to kiwi-wasm.wasm'); | ||
|
|
||
| const kiwi = await builder.build({ | ||
| modelFiles: { | ||
| 'combiningRule.txt': '/path/to/model/combiningRule.txt', | ||
| 'default.dict': '/path/to/model/default.dict', | ||
| 'extract.mdl': '/path/to/model/extract.mdl', | ||
| 'multi.dict': '/path/to/model/multi.dict', | ||
| 'sj.knlm': '/path/to/model/sj.knlm', | ||
| 'sj.morph': '/path/to/model/sj.morph', | ||
| 'skipbigram.mdl': '/path/to/model/skipbigram.mdl', | ||
| 'typo.dict': '/path/to/model/typo.dict', | ||
| } | ||
| }); | ||
|
|
||
| const tokens = kiwi.analyze('다음은 예시 텍스트입니다.', Match.allWithNormalizing); | ||
| /* Output: { | ||
| "score": -39.772212982177734, | ||
| "tokens": [ | ||
| { | ||
| "length": 2, | ||
| "lineNumber": 0, | ||
| "pairedToken": 4294967295, | ||
| "position": 0, | ||
| "score": -6.5904083251953125, | ||
| "sentPosition": 0, | ||
| "str": "다음", | ||
| "subSentPosition": 0, | ||
| "tag": "NNG", | ||
| "typoCost": 0, | ||
| "typoFormId": 0, | ||
| "wordPosition": 0 | ||
| }, | ||
| { | ||
| "length": 1, | ||
| "lineNumber": 0, | ||
| "pairedToken": 4294967295, | ||
| "position": 2, | ||
| "score": -1.844599723815918, | ||
| "sentPosition": 0, | ||
| "str": "은", | ||
| "subSentPosition": 0, | ||
| "tag": "JX", | ||
| "typoCost": 0, | ||
| "typoFormId": 0, | ||
| "wordPosition": 0 | ||
| }, | ||
| ... | ||
| ] | ||
| } */ | ||
| } | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -e | ||
|
|
||
| # Change to the script directory | ||
| cd "$(dirname "$0")" | ||
|
|
||
| # Get root directory | ||
| REPO_ROOT_DIR=$(git rev-parse --show-toplevel) | ||
|
|
||
| # Check if emscripten is installed | ||
| if ! command -v emcmake &> /dev/null; then | ||
| echo "Emscripten is not installed. Please install it and make sure it is in your PATH." | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Generate the package structure | ||
| mkdir -p package/src/build | ||
| mkdir -p package/dist | ||
|
|
||
| # Find core count for make. Prefer nproc, then sysctl, then default to 1 | ||
| if command -v nproc &> /dev/null; then | ||
| CORE_COUNT=$(nproc) | ||
| elif command -v sysctl &> /dev/null; then | ||
| CORE_COUNT=$(sysctl -n hw.logicalcpu) | ||
| else | ||
| CORE_COUNT=1 | ||
| fi | ||
|
|
||
| # Build the wasm module and read the project version | ||
| mkdir -p build | ||
| cd build | ||
| emcmake cmake \ | ||
| -DCMAKE_BUILD_TYPE=Release \ | ||
| -DKIWI_USE_CPUINFO=OFF \ | ||
| -DKIWI_USE_MIMALLOC=OFF \ | ||
| -DKIWI_BUILD_TEST=OFF \ | ||
| -DKIWI_BUILD_CLI=OFF \ | ||
| -DKIWI_BUILD_EVALUATOR=OFF \ | ||
| -DKIWI_BUILD_MODEL_BUILDER=OFF \ | ||
| $REPO_ROOT_DIR | ||
| make -j $CORE_COUNT | ||
| PROJECT_VERSION=$(grep -m 1 CMAKE_PROJECT_VERSION:STATIC CMakeCache.txt | cut -d'=' -f2) | ||
| if [ -z "$PROJECT_VERSION" ]; then | ||
| echo "Failed to read project version from CMakeCache.txt" | ||
| exit 1 | ||
| fi | ||
| cd .. | ||
|
|
||
| # Copy the generated files to the package | ||
| cp build/bindings/wasm/kiwi-wasm.js package/src/build/kiwi-wasm.js | ||
| cp build/bindings/wasm/kiwi-wasm.wasm package/dist/kiwi-wasm.wasm | ||
|
|
||
| # Build typescript wrapper package and update the version | ||
| cd package | ||
| npm install | ||
| npm run build | ||
| npm version --no-git-tag-version --allow-same-version $PROJECT_VERSION | ||
| cd .. | ||
|
|
||
| # Build the demo package if --demo or --demo-dev is passed | ||
| # --demo with create a static build | ||
| # --demo-dev will start a development server | ||
| if [ "$1" == "--demo" ] || [ "$1" == "--demo-dev" ]; then | ||
| cd package-demo | ||
| npm install | ||
| if [ "$1" == "--demo-dev" ]; then | ||
| npm run dev | ||
| else | ||
| npm run build | ||
| fi | ||
| cd .. | ||
| fi |
Oops, something went wrong.