Merge pull request #37 from am9zZWY/tokenization

Tokenization
am9zZWY · Jul 17, 2024 · 02b75f7 · 02b75f7
2 parents b302f9c + ccd9fa9
commit 02b75f7
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 16 deletions.
diff --git a/engine/custom_rank.py b/engine/custom_rank.py
@@ -3,12 +3,13 @@
 import pandas as pd
 
 from custom_db import get_doc_by_id
-from custom_tokenizer import tokenize_data
 from sklearn.feature_extraction.text import TfidfVectorizer
 
+from tokenizer import process_text
+
 
 def preprocess_query(Q):
-    tokenized_query = tokenize_data(Q)
+    tokenized_query = process_text(Q)
     return tokenized_query
 
 

diff --git a/engine/main.py b/engine/main.py
@@ -14,7 +14,7 @@
 # Pipeline
 from crawl import Crawler
 from custom_db import index_pages, access_index, save_pages
-from custom_tokenizer import Tokenizer
+from tokenizer import Tokenizer
 from index import Indexer
 
 # Threading

diff --git a/engine/requirements.txt b/engine/requirements.txt
@@ -12,3 +12,4 @@ pandas==2.2.2
 scikit-learn==1.5.1
 aiohttp==3.9.5
 spacy==3.7.5
+lxml==5.2.2
diff --git a/engine/test.py b/engine/test.py
diff --git a/engine/tokenizer.py b/engine/tokenizer.py
@@ -257,7 +257,9 @@ async def process(self, data, link):
     "I'm 6'2\" tall and I weigh 180 lbs. I'm 25 years old.",
 ]
 
-for sentence in test_sentences:
-    print(f"Original: {sentence}")
-    print(f"Tokenized: {process_text(sentence)}")
-    print()
+if __name__ == "__main__":
+
+    for sentence in test_sentences:
+        print(f"Original: {sentence}")
+        print(f"Tokenized: {process_text(sentence)}")
+        print()