Adapt files to changed imports

am9zZWY · Jul 17, 2024 · 80dcdf2 · 80dcdf2
1 parent 23f59b0
commit 80dcdf2
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 12 deletions.
diff --git a/engine/custom_rank.py b/engine/custom_rank.py
@@ -3,12 +3,13 @@
 import pandas as pd
 
 from custom_db import get_doc_by_id
-from custom_tokenizer import tokenize_data
 from sklearn.feature_extraction.text import TfidfVectorizer
 
+from tokenizer import process_text
+
 
 def preprocess_query(Q):
-    tokenized_query = tokenize_data(Q)
+    tokenized_query = process_text(Q)
     return tokenized_query
 
 

diff --git a/engine/main.py b/engine/main.py
@@ -14,7 +14,7 @@
 # Pipeline
 from crawl import Crawler
 from custom_db import index_pages, access_index, save_pages
-from custom_tokenizer import Tokenizer
+from tokenizer import Tokenizer
 from index import Indexer
 
 # Threading

diff --git a/engine/test.py b/engine/test.py