Tokenization

engine/custom_rank.py

@@ -3,12 +3,13 @@
 import pandas as pd
 
 from custom_db import get_doc_by_id
-from custom_tokenizer import tokenize_data
 from sklearn.feature_extraction.text import TfidfVectorizer
 
+from tokenizer import process_text
+
 
 def preprocess_query(Q):
-    tokenized_query = tokenize_data(Q)
+    tokenized_query = process_text(Q)
     return tokenized_query
 
 

engine/main.py

@@ -14,7 +14,7 @@
 # Pipeline
 from crawl import Crawler
 from custom_db import index_pages, access_index, save_pages
-from custom_tokenizer import Tokenizer
+from tokenizer import Tokenizer
 from index import Indexer
 
 # Threading

engine/requirements.txt

@@ -12,3 +12,4 @@ pandas==2.2.2
 scikit-learn==1.5.1
 aiohttp==3.9.5
 spacy==3.7.5
+lxml==5.2.2

engine/tokenizer.py

@@ -257,7 +257,9 @@ async def process(self, data, link):
     "I'm 6'2\" tall and I weigh 180 lbs. I'm 25 years old.",
 ]
 
-for sentence in test_sentences:
-    print(f"Original: {sentence}")
-    print(f"Tokenized: {process_text(sentence)}")
-    print()
+if __name__ == "__main__":
+
+    for sentence in test_sentences:
+        print(f"Original: {sentence}")
+        print(f"Tokenized: {process_text(sentence)}")
+        print()