Fix: spaces are highlighted by mistake

Endle · Oct 22, 2024 · 7c4a3d1 · 7c4a3d1
1 parent 2ea9ca0
commit 7c4a3d1
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/fire_seq_search_server/debug_server.sh b/fire_seq_search_server/debug_server.sh
@@ -4,7 +4,7 @@ rm -f ./fire_seq_search_server
 cargo build --features llm
 cp  target/debug/fire_seq_search_server ./fire_seq_search_server
 
-export RUST_LOG="warn,fire_seq_search_server=debug"
+export RUST_LOG="warn,fire_seq_search_server=info"
 #export RUST_LOG="debug"
 export RUST_BACKTRACE=1
 #RAYON_NUM_THREADS=1 

diff --git a/fire_seq_search_server/src/language_tools/tokenizer.rs b/fire_seq_search_server/src/language_tools/tokenizer.rs
@@ -13,6 +13,7 @@ use log::{debug, info};
 pub fn filter_out_stopwords<'a,'b>(term_tokens: &'a [String], nltk: &'b HashSet<String>) -> Vec<&'a str> {
     let term_ref: Vec<&str> = term_tokens.iter().map(|s| &**s).collect();
     let terms_selected: Vec<&str> = term_ref.into_iter()
+        .filter(|&s| ! (s.trim().is_empty()  ) )
         .filter(|&s| !nltk.contains(&(&s).to_lowercase()  )  )
         .collect();
     terms_selected