Add main for tokenizer

am9zZWY · Jul 17, 2024 · ccd9fa9 · ccd9fa9
1 parent 80dcdf2
commit ccd9fa9
Showing 1 changed file with 6 additions and 4 deletions.
diff --git a/engine/tokenizer.py b/engine/tokenizer.py
@@ -257,7 +257,9 @@ async def process(self, data, link):
     "I'm 6'2\" tall and I weigh 180 lbs. I'm 25 years old.",
 ]
 
-for sentence in test_sentences:
-    print(f"Original: {sentence}")
-    print(f"Tokenized: {process_text(sentence)}")
-    print()
+if __name__ == "__main__":
+
+    for sentence in test_sentences:
+        print(f"Original: {sentence}")
+        print(f"Tokenized: {process_text(sentence)}")
+        print()