update o200k_base regex too

anysphere · Nov 9, 2024 · fc753fa · fc753fa
1 parent c9aaf1b
commit fc753fa
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/openai_public.rs b/src/openai_public.rs
@@ -149,7 +149,7 @@ impl EncodingFactory {
         r"\p{N}{1,3}",
         r" ?[^\s\p{L}\p{N}]+[\r\n/]*",
         r"\s*[\r\n]+",
-        r"\s+(?!\S)",
+        r"\s+$",
         r"\s+",
     ].join("|");