Added JavaScript/wasm target via emscripten

.github/workflows/release.yml

@@ -326,3 +326,21 @@ jobs:
         asset_path: artifacts/kiwi-java.jar
         asset_name: kiwi-java-${{ steps.get_release.outputs.tag_name }}-lnx-${{ matrix.arch }}.jar
         asset_content_type: application/octet-stream
+
+  build-emscripten:
+    name: Emscripten
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+        lfs: true
+    - uses: mymindstorm/setup-emsdk@v14
+    - name: Build
+      run: |
+        cd bindings/wasm
+        ./build.sh
+    - uses: JS-DevTools/npm-publish@v3
+      with:
+        token: ${{ secrets.NPM_TOKEN }}
+        package: bindings/wasm/package

CMakeLists.txt

@@ -8,6 +8,9 @@ set ( CMAKE_VERBOSE_MAKEFILE true )
 option(KIWI_USE_MIMALLOC  "Use mimalloc for faster memory allocation" ON)
 option(KIWI_USE_CPUINFO  "Use cpuinfo for dynamic CPU dispatching" ON)
 option(KIWI_STATIC_WITHOUT_MT  "Use /MT Option in building kiwi_static" OFF)
+option(KIWI_BUILD_CLI  "Build CLI tool" ON)
+option(KIWI_BUILD_EVALUATOR  "Build Evaluator" ON)
+option(KIWI_BUILD_MODEL_BUILDER  "Build Model Builder" ON)
 option(KIWI_BUILD_TEST  "Build Test sets" ON)
 option(KIWI_JAVA_BINDING  "Build Java binding" OFF)
 set(KIWI_CPU_ARCH "" CACHE STRING "Set architecture type for macOS")
@@ -23,7 +26,9 @@ if (NOT CMAKE_BUILD_TYPE)
 endif()
 
 if(NOT KIWI_CPU_ARCH)
-  if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
+  if (EMSCRIPTEN)
+    set(KIWI_CPU_ARCH "wasm")
+  elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
     set(KIWI_CPU_ARCH "x86_64")
   elseif (HOST_ARCHITECTURE MATCHES "^arm64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm|aarch64)")
     set(KIWI_CPU_ARCH "arm64")
@@ -174,6 +179,8 @@ elseif (KIWI_CPU_ARCH MATCHES "arm64")
     src/archImpl/neon.cpp
   )
   set_source_files_properties(src/archImpl/neon.cpp PROPERTIES COMPILE_FLAGS "-march=armv8-a")
+elseif (KIWI_CPU_ARCH MATCHES "wasm")
+  message("Compiling for wasm")
 else()
   message("Compiling for other")
 endif()
@@ -208,30 +215,36 @@ target_compile_options("${PROJECT_NAME}" PRIVATE "${ADDITIONAL_FLAGS}")
 #target_link_libraries("${PROJECT_NAME}_static" cpuinfo_internals)
 #target_link_libraries("${PROJECT_NAME}" cpuinfo)
 
-add_executable( "${PROJECT_NAME}-cli-${PROJECT_VERSION}"
-  tools/runner.cpp
-)
+if (KIWI_BUILD_CLI)
+  add_executable( "${PROJECT_NAME}-cli-${PROJECT_VERSION}"
+    tools/runner.cpp
+  )
 
-target_link_libraries( "${PROJECT_NAME}-cli-${PROJECT_VERSION}"
-  "${PROJECT_NAME}_static"
-)
+  target_link_libraries( "${PROJECT_NAME}-cli-${PROJECT_VERSION}"
+    "${PROJECT_NAME}_static"
+  )
+endif()
 
-add_executable( "${PROJECT_NAME}-evaluator"
-  tools/Evaluator.cpp
-  tools/evaluator_main.cpp
-)
+if (KIWI_BUILD_EVALUATOR)
+  add_executable( "${PROJECT_NAME}-evaluator"
+    tools/Evaluator.cpp
+    tools/evaluator_main.cpp
+  )
 
-target_link_libraries( "${PROJECT_NAME}-evaluator"
-  "${PROJECT_NAME}_static"
-)
+  target_link_libraries( "${PROJECT_NAME}-evaluator"
+    "${PROJECT_NAME}_static"
+  )
+endif()
 
-add_executable( "${PROJECT_NAME}-model-builder"
-  tools/model_builder.cpp
-)
+if (KIWI_BUILD_MODEL_BUILDER)
+  add_executable( "${PROJECT_NAME}-model-builder"
+    tools/model_builder.cpp
+  )
 
-target_link_libraries( "${PROJECT_NAME}-model-builder"
-  "${PROJECT_NAME}_static"
-)
+  target_link_libraries( "${PROJECT_NAME}-model-builder"
+    "${PROJECT_NAME}_static"
+  )
+endif()
 
 if(MSVC)
   if(KIWI_STATIC_WITHOUT_MT)
@@ -262,13 +275,17 @@ if(UNIX AND NOT APPLE)
     rt
   )
 
-  target_link_libraries( "${PROJECT_NAME}-cli-${PROJECT_VERSION}"
-    rt
-  )
+  if (KIWI_BUILD_CLI)
+    target_link_libraries( "${PROJECT_NAME}-cli-${PROJECT_VERSION}"
+      rt
+    )
+  endif()
 
-  target_link_libraries( "${PROJECT_NAME}-evaluator"
-    rt
-  )
+  if (KIWI_BUILD_EVALUATOR)
+    target_link_libraries( "${PROJECT_NAME}-evaluator"
+      rt
+    )
+  endif()
 endif()
 
 target_compile_definitions("${PROJECT_NAME}"
@@ -291,3 +308,7 @@ endif()
 if(KIWI_JAVA_BINDING)
   add_subdirectory( bindings/java )
 endif()
+
+if(EMSCRIPTEN)
+  add_subdirectory( bindings/wasm )
+endif()

bindings/wasm/CMakeLists.txt

@@ -0,0 +1,11 @@
+add_executable( "${PROJECT_NAME}-wasm"
+  kiwi_wasm.cpp
+)
+
+target_link_libraries( "${PROJECT_NAME}-wasm"
+  "${PROJECT_NAME}_static"
+)
+
+set_target_properties("${PROJECT_NAME}-wasm" PROPERTIES
+  LINK_FLAGS "--bind -s WASM=1 -s ALLOW_MEMORY_GROWTH=1 -s EXPORT_ES6=1 -s MODULARIZE=1 -s EXPORT_NAME=kiwi -s 'EXPORTED_RUNTIME_METHODS=[\"FS\"]'"
+)

bindings/wasm/README.md

@@ -0,0 +1,79 @@
+# kiwi-nlp, 한국어 형태소 분석기 Kiwi의 TypeScript/JavaScript 바인딩
+
+## Building
+
+Additionally to the requirements of the main project, you need to install [Emscripten](https://emscripten.org/docs/getting_started/downloads.html) and [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm).
+
+To build the package, simply run `./build.sh`.
+
+This is currently only supported on Linux and macOS. You can run the build script on Windows by using [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
+
+You can pass the `--demo` flag to build the demo in `package-demo` as well.
+If you pass `--demo-dev`, a development server for the demo will be started.
+
+Running the above command also automatically upgrades to package version if it doesn't match the version in the main project.
+
+## Documentation
+
+The documentation for the package can be generated by running `npm run doc` inside the `package` directory.
+
+The main entry point for the API is `KiwiBuilder`, which is used to create instances `Kiwi`.
+
+## Example Usage
+
+```javascript
+import { KiwiBuilder, Match } from 'kiwi-nlp';
+
+async function example() {
+    const builder = await KiwiBuilder.create('path to kiwi-wasm.wasm');
+
+    const kiwi = await builder.build({
+        modelFiles: {
+            'combiningRule.txt': '/path/to/model/combiningRule.txt',
+            'default.dict': '/path/to/model/default.dict',
+            'extract.mdl': '/path/to/model/extract.mdl',
+            'multi.dict': '/path/to/model/multi.dict',
+            'sj.knlm': '/path/to/model/sj.knlm',
+            'sj.morph': '/path/to/model/sj.morph',
+            'skipbigram.mdl': '/path/to/model/skipbigram.mdl',
+            'typo.dict': '/path/to/model/typo.dict',
+        }
+    });
+
+    const tokens = kiwi.analyze('다음은 예시 텍스트입니다.', Match.allWithNormalizing);
+    /* Output: {
+        "score": -39.772212982177734,
+        "tokens": [
+            {
+                "length": 2,
+                "lineNumber": 0,
+                "pairedToken": 4294967295,
+                "position": 0,
+                "score": -6.5904083251953125,
+                "sentPosition": 0,
+                "str": "다음",
+                "subSentPosition": 0,
+                "tag": "NNG",
+                "typoCost": 0,
+                "typoFormId": 0,
+                "wordPosition": 0
+            },
+            {
+                "length": 1,
+                "lineNumber": 0,
+                "pairedToken": 4294967295,
+                "position": 2,
+                "score": -1.844599723815918,
+                "sentPosition": 0,
+                "str": "은",
+                "subSentPosition": 0,
+                "tag": "JX",
+                "typoCost": 0,
+                "typoFormId": 0,
+                "wordPosition": 0
+            },
+            ...
+        ]
+    } */
+}
+```

bindings/wasm/build.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Change to the script directory
+cd "$(dirname "$0")"
+
+# Get root directory
+REPO_ROOT_DIR=$(git rev-parse --show-toplevel)
+
+# Check if emscripten is installed
+if ! command -v emcmake &> /dev/null; then
+    echo "Emscripten is not installed. Please install it and make sure it is in your PATH."
+    exit 1
+fi
+
+# Generate the package structure
+mkdir -p package/src/build
+mkdir -p package/dist
+
+# Find core count for make. Prefer nproc, then sysctl, then default to 1
+if command -v nproc &> /dev/null; then
+    CORE_COUNT=$(nproc)
+elif command -v sysctl &> /dev/null; then
+    CORE_COUNT=$(sysctl -n hw.logicalcpu)
+else
+    CORE_COUNT=1
+fi
+
+# Build the wasm module and read the project version
+mkdir -p build
+cd build
+emcmake cmake \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DKIWI_USE_CPUINFO=OFF \
+    -DKIWI_USE_MIMALLOC=OFF \
+    -DKIWI_BUILD_TEST=OFF \
+    -DKIWI_BUILD_CLI=OFF \
+    -DKIWI_BUILD_EVALUATOR=OFF \
+    -DKIWI_BUILD_MODEL_BUILDER=OFF \
+    $REPO_ROOT_DIR
+make -j $CORE_COUNT
+PROJECT_VERSION=$(grep -m 1 CMAKE_PROJECT_VERSION:STATIC CMakeCache.txt | cut -d'=' -f2)
+if [ -z "$PROJECT_VERSION" ]; then
+    echo "Failed to read project version from CMakeCache.txt"
+    exit 1
+fi
+cd ..
+
+# Copy the generated files to the package
+cp build/bindings/wasm/kiwi-wasm.js package/src/build/kiwi-wasm.js
+cp build/bindings/wasm/kiwi-wasm.wasm package/dist/kiwi-wasm.wasm
+
+# Build typescript wrapper package and update the version
+cd package
+npm install
+npm run build
+npm version --no-git-tag-version --allow-same-version $PROJECT_VERSION
+cd ..
+
+# Build the demo package if --demo or --demo-dev is passed
+# --demo with create a static build
+# --demo-dev will start a development server
+if [ "$1" == "--demo" ] || [ "$1" == "--demo-dev" ]; then
+    cd package-demo
+    npm install
+    if [ "$1" == "--demo-dev" ]; then
+        npm run dev
+    else
+        npm run build
+    fi
+    cd ..
+fi