Merge pull request #38 from snipsco/release/0.6.1

Release 0.6.1

.travis.yml

@@ -1,60 +1,35 @@
-language: python
-
 matrix:
   include:
     - os: osx
       osx_image: xcode8
       language: generic
       env:
         - TOXENV=py27
+        - PYTHON_TESTS=true
     - os: osx
       osx_image: xcode8
       language: generic
       env:
         - TOXENV=py36
+        - PYTHON_TESTS=true
     - os: linux
+      language: python
       python: 2.7
       env:
         - TOXENV=py27
+        - PYTHON_TESTS=true
     - os: linux
+      language: python
       python: 3.6
       env:
         - TOXENV=py36
+        - PYTHON_TESTS=true
+    - os: linux
+      language: rust
+      rust: stable
+      env:
+        - RUST_TESTS=true
 
-before_install: |
-  # Install Rust
-  curl https://sh.rustup.rs -sSf | bash -s -- -y
-  if [ "$TRAVIS_OS_NAME" == "osx" ]; then
-    brew update || brew update
-
-    brew outdated openssl || brew upgrade openssl
-    brew install [email protected]
-
-    # install pyenv
-    git clone --depth 1 https://github.com/pyenv/pyenv ~/.pyenv
-    PYENV_ROOT="$HOME/.pyenv"
-    PATH="$PYENV_ROOT/bin:$PATH"
-    eval "$(pyenv init -)"
-
-    case "${TOXENV}" in
-      py27)
-        curl -O https://bootstrap.pypa.io/get-pip.py
-        python get-pip.py --user
-        ;;
-      py36)
-        pyenv install 3.6.1
-        pyenv global 3.6.1
-        ;;
-    esac
-    pyenv rehash
-    python -m pip install --user virtualenv
-
-    # A manual check that the correct version of Python is running.
-    python --version
-  fi
-
-install:
-  - ./.travis/install.sh
+before_install: . ./.travis/before_install.sh
 
-script:
-  - ./.travis/test.sh
+script: ./.travis/test.sh

.travis/before_install.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Install Rust
+if [ -z ${TRAVIS_RUST_VERSION+w} ]; then
+  curl https://sh.rustup.rs -sSf | bash -s -- -y
+fi
+
+if [ $TRAVIS_OS_NAME == "osx" ] && [ $PYTHON_TESTS == "true" ]; then
+  # install pyenv
+  git clone --depth 1 https://github.com/pyenv/pyenv ~/.pyenv
+  PYENV_ROOT="$HOME/.pyenv"
+  PATH="$PYENV_ROOT/bin:$PATH"
+  eval "$(pyenv init -)"
+
+  case "${TOXENV}" in
+    "py27")
+      pyenv install 2.7.14
+      pyenv global 2.7.14
+      ;;
+    "py36")
+      pyenv install 3.6.1
+      pyenv global 3.6.1
+      ;;
+  esac
+  pyenv rehash
+
+  # A manual check that the correct version of Python is running.
+  python --version
+fi

.travis/common.sh

@@ -16,15 +16,3 @@ has() {
 	local item=$1; shift
 	echo " $@ " | grep -q " $(escape $item) "
 }
-
-parseRustVersion() {
-    grep -w Cargo.toml -e '^version = ".*' | sed -- 's/version = "//g' | sed -- 's/"//g'
-}
-
-updateVersions() {
-    local tagVersion=$1
-    echo "Updating version..."
-    ./update_version.sh ${tagVersion} || die "Could not upload version"
-}
-
-TAG_VERSION=$(parseRustVersion)

.travis/test.sh

@@ -1,12 +1,18 @@
 #!/usr/bin/env bash
 source .travis/common.sh
 
-echo "Rust tests..."
 export PATH="/usr/local/bin:$HOME/.cargo/bin:$PATH"
-cargo test --all || die "Rust tests failed"
-
-echo "Python tests..."
 perl -p -i -e "s/^snips-nlu-utils = .*\$/snips-nlu-utils = { path = \"..\/..\" \}/g" */**/Cargo.toml
-cd python
-python -m pip install tox
-tox || die "Python tests failed"
+
+if [ "${RUST_TESTS}" == "true" ]; then
+  echo "Rust tests..."
+  cargo test --all || die "Rust tests failed"
+fi
+
+if [ "${PYTHON_TESTS}" == "true" ]; then
+  echo "Python tests..."
+  cd python
+  python -m pip install tox
+  tox || die "Python tests failed"
+  cd ..
+fi

CHANGELOG.md

@@ -0,0 +1,9 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+## [0.6.1] - 2018-04-13
+
+### Added
+- Add `get_shape` function
+
+[0.6.1]: https://github.com/snipsco/snips-nlu-utils/compare/0.6.0...0.6.1

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-utils"
-version = "0.6.0"
+version = "0.6.1"
 authors = ["Adrien Ball <[email protected]>"]
 
 [dependencies]

README.rst

@@ -4,11 +4,22 @@ Snips NLU Utils
 .. image:: https://travis-ci.org/snipsco/snips-nlu-utils.svg?branch=master
     :target: https://travis-ci.org/snipsco/snips-nlu-utils
 
+
 Rust library for NLU utils with wrappers in other languages.
 
+Installation
+------------
+
+Add it to your ``Cargo.toml``:
+
+.. code-block:: toml
+
+   [dependencies]
+   snips-nlu-utils = { git = "https://github.com/snipsco/snips-nlu-utils", branch = "master" }
+
+Add ``extern crate snips_nlu_utils`` to your crate root and you are good to go!
+
 Python wrapper
 --------------
 
-See the `README`_ of the python wrapper for installation instruction.
-
-.. _README: https://github.com/snipsco/snips-nlu-utils/blob/master/python/README.rst
+See the `README <python/README.rst>`_ of the python wrapper for installation instruction.

python/README.rst

@@ -1,6 +1,16 @@
 Snips NLU utils Python wrapper
 ==============================
 
+.. image:: https://travis-ci.org/snipsco/snips-nlu-utils.svg?branch=master
+    :target: https://travis-ci.org/snipsco/snips-nlu-utils
+
+.. image:: https://img.shields.io/pypi/v/snips-nlu-utils.svg?branch=master
+    :target: https://pypi.python.org/pypi/snips-nlu-utils
+
+.. image:: https://img.shields.io/pypi/pyversions/snips-nlu-utils.svg?branch=master
+    :target: https://pypi.python.org/pypi/snips-nlu-utils
+
+
 This library is a wrapper of a Rust NLU utils library, which is used by Snips NLU
 
 Installation

python/snips_nlu_utils/__init__.py

@@ -1,2 +1,3 @@
-from ._snips_nlu_utils_py import (tokenize, tokenize_light, compute_all_ngrams,
-                            normalize, remove_diacritics)
+from ._snips_nlu_utils_py import (
+    get_shape, tokenize, tokenize_light, compute_all_ngrams,normalize,
+    remove_diacritics)

python/snips_nlu_utils/__version__

@@ -1 +1 @@
-0.6.0
+0.6.1

python/snips_nlu_utils/tests/test_nlu_utils.py

@@ -1,14 +1,16 @@
+# coding=utf-8
 from __future__ import unicode_literals
 
 import unittest
-
 from builtins import str, bytes
 
-from snips_nlu_utils import tokenize, tokenize_light
+from snips_nlu_utils import (
+    compute_all_ngrams, get_shape, normalize, remove_diacritics, tokenize,
+    tokenize_light)
 
 
 class TestNLUUtils(unittest.TestCase):
-    def test_package_should_tokenize(self):
+    def test_should_tokenize(self):
         # Given
         u = "let's eat food tonight"
         language = "en"
@@ -20,7 +22,7 @@ def test_package_should_tokenize(self):
         self.assertGreater(len(tokens), 0)
         self.assertTrue(all(isinstance(t, dict) for t in tokens))
 
-    def test_package_should_tokenize_light(self):
+    def test_should_tokenize_light(self):
         # Given
         u = "let's eat food tonight"
         language = "en"
@@ -49,3 +51,36 @@ def test_tokenize_light_should_raise_on_string(self):
         # When / Then
         with self.assertRaises(TypeError):
             tokenize_light(s, language)
+
+    def test_should_remove_diacritics(self):
+        self.assertEqual("Hello", remove_diacritics("Hëllo"))
+
+    def test_should_normalize(self):
+        self.assertEqual("hello", normalize("Hëllo"))
+
+    def test_should_compute_all_ngrams(self):
+        # Given
+        tokens = ["hello", "beautiful", "world", "!"]
+
+        # When
+        ngrams = compute_all_ngrams(tokens, 3)
+
+        # Then
+        expected_ngrams = [
+            {'ngram': 'hello', 'token_indexes': [0]},
+            {'ngram': 'hello beautiful', 'token_indexes': [0, 1]},
+            {'ngram': 'hello beautiful world', 'token_indexes': [0, 1, 2]},
+            {'ngram': 'beautiful', 'token_indexes': [1]},
+            {'ngram': 'beautiful world', 'token_indexes': [1, 2]},
+            {'ngram': 'beautiful world !', 'token_indexes': [1, 2, 3]},
+            {'ngram': 'world', 'token_indexes': [2]},
+            {'ngram': 'world !', 'token_indexes': [2, 3]},
+            {'ngram': '!', 'token_indexes': [3]}
+        ]
+        self.assertListEqual(expected_ngrams, ngrams)
+
+    def test_should_get_shape(self):
+        self.assertEqual("xxx", get_shape("hello"))
+        self.assertEqual("XXX", get_shape("HELLO"))
+        self.assertEqual("Xxx", get_shape("Hello"))
+        self.assertEqual("xX", get_shape("hEllo"))

python/snips_nlu_utils_py/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "snips-nlu-utils-py"
-version = "0.6.0"
+version = "0.6.1"
 authors = ["Adrien Ball <[email protected]>"]
 
 [lib]
 name = "_snips_nlu_utils_py"
 crate-type = ["cdylib"]
 
 [dependencies]
-snips-nlu-utils = { git = "https://github.com/snipsco/snips-nlu-utils", tag = "0.6.0" }
+snips-nlu-utils = { git = "https://github.com/snipsco/snips-nlu-utils", tag = "0.6.1" }
 cpython = { version="0.1", default-features=false }

python/snips_nlu_utils_py/src/lib.rs

@@ -7,7 +7,7 @@ pub mod string;
 mod binding_utils;
 
 use cpython::{PyList, PyUnicode};
-use string::{normalize, remove_diacritics};
+use string::{get_shape, normalize, remove_diacritics};
 use token::{tokenize, tokenize_light, compute_all_ngrams};
 
 py_module_initializer!(_snips_nlu_utils_py, init_snips_nlu_utils_py, PyInit__snips_nlu_utils_py, |py, m| {
@@ -16,5 +16,6 @@ py_module_initializer!(_snips_nlu_utils_py, init_snips_nlu_utils_py, PyInit__sni
     m.add(py, "compute_all_ngrams", py_fn!(py, compute_all_ngrams(tokens: PyList, max_ngram_size: i32)))?;
     m.add(py, "normalize", py_fn!(py, normalize(string: PyUnicode)))?;
     m.add(py, "remove_diacritics", py_fn!(py, remove_diacritics(string: PyUnicode)))?;
+    m.add(py, "get_shape", py_fn!(py, get_shape(string: PyUnicode)))?;
     Ok(())
 });

python/snips_nlu_utils_py/src/string.rs

@@ -12,3 +12,8 @@ pub fn remove_diacritics(py: Python, string: PyUnicode) -> PyResult<PyUnicode> {
     let string_without_diacritics = string_utils::remove_diacritics(string.to_string(py)?.borrow());
     return Ok(PyUnicode::new(py, &string_without_diacritics.to_string()));
 }
+
+pub fn get_shape(py: Python, string: PyUnicode) -> PyResult<PyUnicode> {
+    let shape = string_utils::get_shape(string.to_string(py)?.borrow());
+    return Ok(PyUnicode::new(py, &shape.to_string()))
+}

src/string.rs

@@ -103,6 +103,48 @@ fn remove_combination_marks(character: char) -> Option<char> {
     })
 }
 
+
+/// Get the shape of the string in one of the following format:
+///
+/// - "xxx" -> lowercase
+/// - "XXX" -> uppercase
+/// - "Xxx" -> title case
+/// - "xX" -> mixed case
+///
+/// # Examples
+///
+/// ```
+/// use snips_nlu_utils::string::get_shape;
+///
+/// assert_eq!("xxx", get_shape("hello"));
+/// assert_eq!("Xxx", get_shape("Hello"));
+/// assert_eq!("XXX", get_shape("HELLO"));
+/// assert_eq!("xX", get_shape("hEllo"));
+/// ```
+pub fn get_shape(string: &str) -> &'static str {
+    if string.chars().all(char::is_lowercase) {
+        "xxx"
+    } else if string.chars().all(char::is_uppercase) {
+        "XXX"
+    } else if is_title_case(string) {
+        "Xxx"
+    } else {
+        "xX"
+    }
+}
+
+fn is_title_case(string: &str) -> bool {
+    let mut first = true;
+    for c in string.chars() {
+        match (first, c.is_uppercase()) {
+            (true, true) => first = false,
+            (false, false) => continue,
+            _ => return false,
+        }
+    }
+    !first
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -161,4 +203,13 @@ mod tests {
     fn normalize_works() {
         assert_eq!("heloa".to_string(), normalize("  HelöÀ "));
     }
+
+    #[test]
+    fn shape_works() {
+        assert_eq!("xxx", get_shape("hello"));
+        assert_eq!("xxx", get_shape("hëllo"));
+        assert_eq!("Xxx", get_shape("Hello"));
+        assert_eq!("XXX", get_shape("HELLO"));
+        assert_eq!("xX", get_shape("hEllo"));
+    }
 }