fixed bab2min/kiwipiepy#131

bab2min · Aug 20, 2023 · 2f845c6 · 2f845c6
1 parent 2cddd46
commit 2f845c6
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 0 deletions.
diff --git a/include/kiwi/Types.h b/include/kiwi/Types.h
@@ -322,6 +322,8 @@ namespace kiwi
 		{
 			return !operator==(o);
 		}
+
+		uint32_t endPos() const { return position + length; }
 	};
 
 	struct BasicToken

diff --git a/src/Kiwi.cpp b/src/Kiwi.cpp
@@ -482,15 +482,32 @@ namespace kiwi
 			auto& t = tokens[i];
 			if ((i >= nestedEnd) && sp.next(t, nlPos, nestedSentEnd && i == nestedSentEnd))
 			{
+				bool includePrevToken = i > 1 && 
+					(tokens[i - 1].tag == POSTag::so 
+						|| tokens[i - 1].tag == POSTag::sw 
+						|| tokens[i - 1].tag == POSTag::sp 
+						|| tokens[i - 1].tag == POSTag::se)
+					&& tokens[i - 1].endPos() == tokens[i].position
+					&& tokens[i - 1].position > tokens[i - 2].endPos();
 				if (nestedSentEnd)
 				{
 					subSentPos++;
 					accumSubSent++;
+					if (includePrevToken)
+					{
+						tokens[i - 1].subSentPosition = subSentPos;
+					}
 				}
 				else
 				{
 					sentPos++;
 					accumSubSent = 1;
+					if (includePrevToken)
+					{
+						tokens[i - 1].sentPosition = sentPos;
+						tokens[i - 1].wordPosition = 0;
+						accumWordPos = 0;
+					}
 				}
 			}