feat: added support for sanskrit diacriticals (#503)

oramasearch · Oct 1, 2023 · 459f931 · 459f931 · vercel · Oct 1, 2023
1 parent e690808
commit 459f931
Show file tree

Hide file tree

Showing 7 changed files with 73 additions and 0 deletions.
diff --git a/packages/orama/src/components/tokenizer/languages.ts b/packages/orama/src/components/tokenizer/languages.ts
@@ -27,6 +27,7 @@ export const STEMMERS: Record<string, string> = {
   tamil: 'ta',
   turkish: 'tr',
   ukrainian: 'uk',
+  sanskrit: 'sk',
 }
 
 export const SPLITTERS: Record<Language, RegExp> = {
@@ -58,6 +59,7 @@ export const SPLITTERS: Record<Language, RegExp> = {
   slovenian: /[^a-z0-9čžšČŽŠ]+/gim,
   bulgarian: /[^a-z0-9а-яА-Я]+/gim,
   tamil: /[^a-z0-9அ-ஹ]+/gim,
+  sanskrit: /[^a-z0-9A-Zāīūṛḷṃṁḥśṣṭḍṇṅñḻḹṝ]+/gim
 }
 
 export const SUPPORTED_LANGUAGES = Object.keys(STEMMERS)

diff --git a/packages/stemmers/lib/sk.js b/packages/stemmers/lib/sk.js
@@ -0,0 +1,25 @@
+/**
+ * Light Stemmer class for Sanskrit
+ */
+
+class SanskritStemmer {
+  constructor() {
+    this.suffixes = ['aḥ', 'āḥ', 'iḥ', 'īḥ', 'uḥ', 'ūḥ', 'am', 'ām', 'im', 'īm', 'um', 'ūm', 'an', 'ān', 'in', 'īn', 'un', 'ūn', 'as', 'ās', 'is', 'īs', 'us', 'ūs'];
+  }
+
+  stem(word) {
+    for (const suffix of this.suffixes) {
+      if (word.endsWith(suffix)) {
+        return word.slice(0, -suffix.length);
+      }
+    }
+
+    return word;
+  }
+}
+
+const stemmerInstance = new SanskritStemmer();
+
+export function stemmer(word) {
+  return stemmerInstance.stem(word)
+}
diff --git a/packages/stemmers/package.json b/packages/stemmers/package.json
@@ -146,6 +146,11 @@
       "types": "./dist/uk.d.ts",
       "import": "./dist/uk.js",
       "require": "./dist/uk.cjs"
+    },
+    "./sanskrit": {
+      "types": "./dist/sk.d.ts",
+      "import": "./dist/sk.js",
+      "require": "./dist/sk.cjs"
     }
   },
   "files": [

diff --git a/packages/stemmers/scripts/build.js b/packages/stemmers/scripts/build.js
@@ -35,6 +35,7 @@ const stemmers = {
   tamil: 'ta',
   turkish: 'tr',
   ukrainian: 'uk',
+  sanskrit: 'sk',
 }
 
 async function compile(lang, jsExtension, tsExtension, moduleType) {

diff --git a/packages/stopwords/lib/sk.js b/packages/stopwords/lib/sk.js
@@ -0,0 +1,34 @@
+export const stopwords = [
+    'ahaṃ',
+    'tava',
+    'tvayi',
+    'svayam',
+    'vayam',
+    'asmān',
+    'nas',
+    'yat',
+    'yaḥ',
+    'kiṃ',
+    'kaḥ',
+    'saḥ',
+    'taḥ',
+    'tasya',
+    'tasmai',
+    'asya',
+    'tat',
+    'tad',
+    'tatra',
+    'katham',
+    'yadi',
+    'vā',
+    'athavā',
+    'evaṃ',
+    'na',
+    'api',
+    'atha',
+    'sama',
+    'santu',
+    'antaḥ',
+    'antar',
+    'ubhau',
+];
diff --git a/packages/stopwords/package.json b/packages/stopwords/package.json
@@ -146,6 +146,11 @@
       "types": "./dist/uk.d.ts",
       "import": "./dist/uk.js",
       "require": "./dist/uk.cjs"
+    },
+    "./sanskrit": {
+      "types": "./dist/sk.d.ts",
+      "import": "./dist/sk.js",
+      "require": "./dist/sk.cjs"
     }
   },
   "files": [

diff --git a/packages/stopwords/scripts/build.js b/packages/stopwords/scripts/build.js
@@ -35,6 +35,7 @@ const stemmers = {
   tamil: 'ta',
   turkish: 'tr',
   ukrainian: 'uk',
+  sanskrit: 'sk',
 }
 
 async function compile(lang, jsExtension, tsExtension, moduleType) {