Fix infinite loop bug (issue #99) (#101) (#102)

ikawaha · May 17, 2017 · d863ec0 · d863ec0
1 parent 26439f7
commit d863ec0
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 1 deletion.
diff --git a/internal/lattice/lattice.go b/internal/lattice/lattice.go
@@ -146,7 +146,12 @@ func (la *Lattice) Build(inp string) {
 		// (3) UNKNOWN DIC
 		class := la.dic.CharacterCategory(ch)
 		if !anyMatches || la.dic.InvokeList[int(class)] {
-			endPos := pos + utf8.RuneLen(ch)
+			var endPos int
+			if ch != utf8.RuneError {
+				endPos = pos + utf8.RuneLen(ch)
+			} else {
+				endPos = pos + 1
+			}
 			unkWordLen := 1
 			if la.dic.GroupList[int(class)] {
 				for i, w, size := endPos, 1, len(inp); i < size; i += w {

diff --git a/internal/lattice/lattice_test.go b/internal/lattice/lattice_test.go
@@ -224,6 +224,30 @@ func TestLatticeBuild05(t *testing.T) {
 	}
 }
 
+func TestLatticeBuildInvalidInput(t *testing.T) {
+
+	la := New(dic.SysDic(), nil)
+	if la == nil {
+		t.Fatal("cannot new a lattice")
+	}
+	defer la.Free()
+
+	inp := "\x96\x7b\x93\xfa" // sjis encoding for '日本'
+	la.Build(inp)
+	if la.Input != inp {
+		t.Errorf("got %v, expected %v", la.Input, inp)
+	}
+	bos := node{ID: -1}
+	eos := node{ID: -1, Start: 4}
+	if len(la.list) != 6 {
+		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
+	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
+		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], bos)
+	} else if len(la.list[len(la.list)-1]) != 1 || *la.list[len(la.list)-1][0] != eos {
+		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[len(la.list)-1][0], eos)
+	}
+}
+
 func TestKanjiOnly01(t *testing.T) {
 	callAndResponse := []struct {
 		in  string