skip levenshtein weights for full emoji strings

divvun · Jan 18, 2025 · 05212a3 · 05212a3
1 parent 7b60b05
commit 05212a3
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 2 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/divvunspell/Cargo.toml b/divvunspell/Cargo.toml
@@ -23,6 +23,7 @@ unic-segment = "0.9.0"
 unic-char-range = "0.9.0"
 unic-char-property = "0.9.0"
 unic-ucd-category = "0.9.0"
+unic-emoji-char = "0.9.0"
 parking_lot = "0.11.2"
 hashbrown = { version = "0.11", features = ["serde"] }
 lifeguard = "0.6.1"

diff --git a/divvunspell/src/speller/mod.rs b/divvunspell/src/speller/mod.rs
@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
 use smol_str::SmolStr;
 use unic_ucd_category::GeneralCategory;
 use unic_segment::Graphemes;
+use unic_emoji_char::is_emoji;
 
 use self::worker::SpellerWorker;
 use crate::speller::suggestion::Suggestion;
@@ -277,7 +278,9 @@ where
                             strsim::damerau_levenshtein(&words[0].as_str(), &word.as_str())
                                 + strsim::damerau_levenshtein(&word.as_str(), sugg.value());
                         let penalty_middle = reweight.mid_penalty * distance as f32;
-                        let additional_weight = penalty_start + penalty_end + penalty_middle;
+                        let additional_weight = if sugg.value.chars().all(|c|
+                            is_emoji(c)) { 0.0 }
+                            else {penalty_start + penalty_end + penalty_middle};
                         log::trace!("Penalty: +{} = {} + {} * {} + {}",
                             additional_weight, penalty_start, distance,
                             reweight.mid_penalty, penalty_end);