Merge pull request #227 from ikawaha/fix/update-uni-dic

ikawaha · Dec 29, 2020 · 9463033 · 9463033
2 parents c93ae10 + 60ac03b
commit 9463033
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 19 deletions.
diff --git a/go.mod b/go.mod
@@ -5,5 +5,5 @@ go 1.15
 require (
 	github.com/ikawaha/kagome-dict v1.0.1
 	github.com/ikawaha/kagome-dict/ipa v1.0.1
-	github.com/ikawaha/kagome-dict/uni v1.0.1
+	github.com/ikawaha/kagome-dict/uni v1.1.0
 )
diff --git a/go.sum b/go.sum
@@ -2,8 +2,8 @@ github.com/ikawaha/kagome-dict v1.0.1 h1:CdGx80JEz+nBUP0+m+47pubzOhURYPRX+zXLRrg
 github.com/ikawaha/kagome-dict v1.0.1/go.mod h1:pbeH4qwp/LVWKNePhU4Mh11w/l8uUSXKxfvNwRkj3NY=
 github.com/ikawaha/kagome-dict/ipa v1.0.1 h1:pE98DI+3e8kqUSL50M7/QE5BDk/o63/o9fDanNtgkN0=
 github.com/ikawaha/kagome-dict/ipa v1.0.1/go.mod h1:6KbSbFnEwIlR65gYAowfwOkbze3bnFH1BILpOEGBkvY=
-github.com/ikawaha/kagome-dict/uni v1.0.1 h1:0HfqEmpHyoRroR8whDsn/pu3LgMljpvURrACxJlk6hs=
-github.com/ikawaha/kagome-dict/uni v1.0.1/go.mod h1:nij0TsdXZ/WauqM9X0XW7VpOx/JkiSkiGicN1bJpx2Q=
+github.com/ikawaha/kagome-dict/uni v1.1.0 h1:b6VKvsjfYT8SB6Lo1LBqDHwIugR02lhkCWFy1Vl/kPI=
+github.com/ikawaha/kagome-dict/uni v1.1.0/go.mod h1:nij0TsdXZ/WauqM9X0XW7VpOx/JkiSkiGicN1bJpx2Q=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/tokenizer/tokenizer_test.go b/tokenizer/tokenizer_test.go
@@ -82,11 +82,7 @@ func Test_AnalyzeEmptyInput(t *testing.T) {
 		t.Fatalf("got %v, expected %v", tokens, expected)
 	}
 	for i, tok := range tokens {
-		if tok.ID != expected[i].ID ||
-			tok.Class != expected[i].Class ||
-			tok.Start != expected[i].Start ||
-			tok.End != expected[i].End ||
-			tok.Surface != expected[i].Surface {
+		if !tok.Equal(expected[i]) {
 			t.Errorf("got %v, expected %v", tok, expected[i])
 		}
 	}
@@ -131,17 +127,13 @@ func Test_AnalyzeUnknown(t *testing.T) {
 	expected := []Token{
 		{ID: -1, Surface: "BOS"},
 		{ID: 34, Surface: "ポポピ", Start: 0, End: 3, Class: TokenClass(lattice.UNKNOWN)},
-		{ID: -1, Surface: "EOS", Start: 3, End: 3},
+		{ID: -1, Surface: "EOS", Start: 3, End: 3, Position: 9},
 	}
 	if len(tokens) != len(expected) {
 		t.Fatalf("got %v, expected %v", tokens, expected)
 	}
 	for i, tok := range tokens {
-		if tok.ID != expected[i].ID ||
-			tok.Class != expected[i].Class ||
-			tok.Start != expected[i].Start ||
-			tok.End != expected[i].End ||
-			tok.Surface != expected[i].Surface {
+		if !tok.Equal(expected[i]) {
 			t.Errorf("got %v, expected %v", tok, expected[i])
 		}
 	}
@@ -204,11 +196,7 @@ func Test_AnalyzeWithSearchModeEmptyInput(t *testing.T) {
 		t.Fatalf("got %v, expected %v", tokens, expected)
 	}
 	for i, tok := range tokens {
-		if tok.ID != expected[i].ID ||
-			tok.Class != expected[i].Class ||
-			tok.Start != expected[i].Start ||
-			tok.End != expected[i].End ||
-			tok.Surface != expected[i].Surface {
+		if !tok.Equal(expected[i]) {
 			t.Errorf("got %v, expected %v", tok, expected[i])
 		}
 	}