Merge pull request #2 from bosondata/feature/allow-nul-bytes

Allow nul bytes

.gitignore

@@ -70,4 +70,5 @@ docs/_build/
 # Pyenv
 .python-version
 
+.pytest_cache/
 simplet2s/_native*

.travis.yml

@@ -4,10 +4,6 @@ rust:
   - stable
   - nightly
 
-cache:
-  directories:
-  - target
-
 services:
   - docker
 
@@ -18,21 +14,20 @@ matrix:
       rust: stable
 
 before_script:
+  - if [ "${TRAVIS_OS_NAME:-}" == "linux" ]; then
+      sudo apt-get install -y python3-pip;
+    fi
   - if [ "${TRAVIS_OS_NAME:-}" == "osx" ]; then
       brew update;
       brew upgrade python;
       brew install libffi;
     fi
-  - sudo easy_install virtualenv
-  - virtualenv -p python3 ~/virtualenv
-  - source ~/virtualenv/bin/activate
-  - export PATH=$HOME/virtualenv/bin:$PATH
-  - pip install 'travis-cargo<0.2' pytest
+  - sudo pip3 install 'travis-cargo<0.2' pytest
 
 script:
   - travis-cargo test
   - travis-cargo --only nightly bench
-  - python setup.py develop
+  - pip3 install --user -e .
   - pytest -v tests
   - |
     if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_RUST_VERSION" == "nightly" ]]; then
@@ -42,15 +37,15 @@ script:
     fi
   - |
     if [[ "${TRAVIS_TAG:-}" != "" && "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_RUST_VERSION" == "stable" ]]; then
-      python setup.py sdist
+      python3 setup.py sdist
       docker run --rm -it -v `pwd`:/io -w /io quay.io/pypa/manylinux1_x86_64 sh manylinux.sh;
       docker run --rm -it -v `pwd`:/io -w /io quay.io/pypa/manylinux1_i686 sh manylinux.sh;
-      pip install twine;
+      sudo pip3 install twine;
       twine upload --skip-existing dist/*;
     fi
   - |
     if [[ "${TRAVIS_TAG:-}" != "" && "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_RUST_VERSION" == "stable" ]]; then
-      pip install -U wheel twine;
-      python setup.py bdist_wheel;
+      sudo pip3 install -U wheel twine;
+      python3 setup.py bdist_wheel;
       twine upload --skip-existing dist/*;
     fi

cabi/Cargo.toml

@@ -2,13 +2,17 @@
 name = "simplet2s-capi"
 version = "0.1.5"
 authors = ["Messense Lv <[email protected]>"]
+publish = false
+
+[dependencies]
+c_fixed_string = "0.2"
 
 [dependencies.simplet2s]
 version = "0.1"
 path = "../"
 
 [build-dependencies]
-cbindgen = "0.5"
+cbindgen = "0.6"
 
 [lib]
 name = "simplet2s"

cabi/src/lib.rs

@@ -1,21 +1,28 @@
 extern crate simplet2s;
+extern crate c_fixed_string;
 
-use std::ffi::{CStr, CString};
 use std::os::raw::c_char;
+use std::mem;
+
+use c_fixed_string::{CFixedStr, CFixedString};
 
 /// Frees a C str.
 #[no_mangle]
-pub unsafe extern "C" fn simplet2s_str_free(s: *mut c_char) {
+pub unsafe extern "C" fn simplet2s_str_free(s: *mut c_char, len: usize) {
     if !s.is_null() {
-        CString::from_raw(s);
+        CFixedString::from_raw_parts(s, len);
     }
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn simplet2s_convert(s: *const c_char) -> *mut c_char {
-    let s = CStr::from_ptr(s).to_str().unwrap();
-    let mut r = simplet2s::convert(s);
+pub unsafe extern "C" fn simplet2s_convert(s: *const c_char, len: usize) -> *mut c_char {
+    let c_str = CFixedStr::from_ptr(s, len);
+    let s = String::from_utf8_lossy(c_str.as_bytes_full());
+    let mut r = simplet2s::convert(&s);
     r.shrink_to_fit();
-    let c_str = CString::new(r).unwrap();
-    c_str.into_raw()
+    let bytes = r.into_bytes();
+    let mut c_str = CFixedString::new(bytes);
+    let ptr = c_str.as_mut_ptr();
+    mem::forget(c_str);
+    ptr
 }

simplet2s/__init__.py

@@ -18,7 +18,10 @@
 def convert(text):
     if isinstance(text, text_type):
         text = text.encode('utf-8')
-    r = lib.simplet2s_convert(text)
-    r = ffi.gc(r, lib.simplet2s_str_free)
-    s = ffi.string(r).decode('utf-8', 'replace')
-    return s
+    text_len = len(text)
+    try:
+        r = lib.simplet2s_convert(text, text_len)
+        s = ffi.unpack(r, text_len).decode('utf-8', 'replace')
+        return s
+    finally:
+        lib.simplet2s_str_free(r, text_len)

tests/test_simplet2s.py

@@ -18,6 +18,9 @@
     [
         ("憂鬱的台灣烏龜", "忧郁的台湾乌龟"),
         ("測試", "测试"),
+        ("\0測試", "\0测试"),
+        ("測\0試", "测\0试"),
+        ("測試\0", "测试\0"),
         ("瞭读liǎo(瞭解)时，简作「了」", "了读liǎo(了解)时，简作「了」"),
         ("而繁體字苧（zhù）是苧麻", "而繁体字苧（zhù）是苎麻"),
         ("西漢有御史大夫兒寬", "西汉有御史大夫兒宽"),