Merge pull request #89 from ikawaha/develop

Add the appengine build tag and some cosmetic change

.travis.yml

@@ -18,7 +18,10 @@ install:
 
 script:
   - go test ./...
+  - go test -tags=appengine ./...
   - cd tokenizer; go test -benchmem -bench .; cd ..
+  - cd tokenizer; go test -tags=appengine -benchmem -bench .; cd ..
   - cd internal/dic; go test -benchmem -bench .; cd ../..
+  - cd internal/dic; go test -tags=appengine -benchmem -bench .; cd ../..
   - /bin/sh ./go-coverall.sh
 

internal/dic/content.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"io"
 	"strings"
-	"unsafe"
 )
 
 const (
@@ -50,11 +49,5 @@ func (c Contents) WriteTo(w io.Writer) (n int64, err error) {
 
 // NewContents creates dictionary contents from byte slice
 func NewContents(b []byte) [][]string {
-	str := *(*string)(unsafe.Pointer(&b))
-	rows := strings.Split(str, rowDelimiter)
-	m := make([][]string, len(rows))
-	for i, r := range rows {
-		m[i] = strings.Split(r, colDelimiter)
-	}
-	return m
+	return newContents(b)
 }

internal/dic/content_appengine.go

@@ -0,0 +1,34 @@
+// Copyright 2015 ikawaha
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// 	You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build appengine
+
+package dic
+
+import (
+	"bytes"
+)
+
+func newContents(b []byte) [][]string {
+	rows := bytes.Split(b, []byte(rowDelimiter))
+	m := make([][]string, len(rows))
+	for i, r := range rows {
+		cols := bytes.Split(r, []byte(colDelimiter))
+		m[i] = make([]string, len(cols))
+		for j, c := range cols {
+			m[i][j] = string(c)
+		}
+	}
+	return m
+}

internal/dic/content_unsafe.go

@@ -0,0 +1,32 @@
+// Copyright 2015 ikawaha
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// 	You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !appengine
+
+package dic
+
+import (
+	"strings"
+	"unsafe"
+)
+
+func newContents(b []byte) [][]string {
+	str := *(*string)(unsafe.Pointer(&b))
+	rows := strings.Split(str, rowDelimiter)
+	m := make([][]string, len(rows))
+	for i, r := range rows {
+		m[i] = strings.Split(r, colDelimiter)
+	}
+	return m
+}

splitter/splitter_test.go

@@ -166,7 +166,7 @@ func Example() {
 	// Output:
 	// 人魚は、南の方の海にばかり棲んでいるのではありません。
 	// 北の海にも棲んでいたのであります。
-	//北方の海うみの色は、青うございました。
-	//あるとき、岩の上に、女の人魚があがって、あたりの景色をながめながら休んでいました。
-	//小川未明作赤い蝋燭と人魚より
+	// 北方の海うみの色は、青うございました。
+	// あるとき、岩の上に、女の人魚があがって、あたりの景色をながめながら休んでいました。
+	// 小川未明作赤い蝋燭と人魚より
 }