Merge pull request #71 from ikawaha/feature/token_features_20160317

Add a function to get a part-of-speech tag
ikawaha · Mar 17, 2016 · d73e145 · d73e145
2 parents 7f10368 + fed9387
commit d73e145
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 4 deletions.
diff --git a/tokenizer/token.go b/tokenizer/token.go
@@ -80,6 +80,15 @@ func (t Token) Features() (features []string) {
 	return
 }
 
+// Pos returns the first element of features.
+func (t Token) Pos() string {
+	f := t.Features()
+	if len(f) < 1 {
+		return ""
+	}
+	return f[0]
+}
+
 // String returns a string representation of a token.
 func (t Token) String() string {
 	return fmt.Sprintf("%v(%v, %v)%v[%v]", t.Surface, t.Start, t.End, t.Class, t.ID)

diff --git a/tokenizer/token_test.go b/tokenizer/token_test.go
@@ -41,7 +41,7 @@ func TestTokenClassString(t *testing.T) {
 	}
 }
 
-func TestFeatures01(t *testing.T) {
+func TestFeaturesAndPos01(t *testing.T) {
 	tok := Token{
 		ID:      0,
 		Class:   TokenClass(lattice.KNOWN),
@@ -56,9 +56,12 @@ func TestFeatures01(t *testing.T) {
 	if !reflect.DeepEqual(f, expected) {
 		t.Errorf("got %v, expected %v", f, expected)
 	}
+	if p := tok.Pos(); p != f[0] {
+		t.Errorf("got %v, expected %v", p, f[0])
+	}
 }
 
-func TestFeatures02(t *testing.T) {
+func TestFeaturesAndPos02(t *testing.T) {
 	tok := Token{
 		ID:      0,
 		Class:   TokenClass(lattice.UNKNOWN),
@@ -73,9 +76,12 @@ func TestFeatures02(t *testing.T) {
 	if !reflect.DeepEqual(f, expected) {
 		t.Errorf("got %v, expected %v", f, expected)
 	}
+	if p := tok.Pos(); p != f[0] {
+		t.Errorf("got %v, expected %v", p, f[0])
+	}
 }
 
-func TestFeatures03(t *testing.T) {
+func TestFeaturesAndPos03(t *testing.T) {
 	tok := Token{
 		ID:      0,
 		Class:   TokenClass(lattice.USER),
@@ -95,9 +101,12 @@ func TestFeatures03(t *testing.T) {
 	if !reflect.DeepEqual(f, expected) {
 		t.Errorf("got %v, expected %v", f, expected)
 	}
+	if p := tok.Pos(); p != f[0] {
+		t.Errorf("got %v, expected %v", p, f[0])
+	}
 }
 
-func TestFeatures04(t *testing.T) {
+func TestFeaturesAndPos04(t *testing.T) {
 	tok := Token{
 		ID:      0,
 		Class:   DUMMY,
@@ -116,6 +125,9 @@ func TestFeatures04(t *testing.T) {
 	if len(f) != 0 {
 		t.Errorf("got %v, expected empty", f)
 	}
+	if p := tok.Pos(); p != "" {
+		t.Errorf("got %v, expected empty", p)
+	}
 }
 
 func TestTokenString01(t *testing.T) {