Skip to content

Commit

Permalink
feat: added CharFilters for open search
Browse files Browse the repository at this point in the history
Refs: SITKO-CORE-T-23
  • Loading branch information
IgorAlymov committed Aug 6, 2024
1 parent f718071 commit 92eb6fc
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 5 deletions.
63 changes: 63 additions & 0 deletions src/Sitko.Core.Search.OpenSearch/OpenSearchHelper.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
namespace Sitko.Core.Search.OpenSearch;

public static class OpenSearchHelper
{
public static readonly string[] RusEnKeys =
[
"a => ф",
"b => и",
"c => с",
"d => в",
"e => у",
"f => а",
"g => п",
"h => р",
"i => ш",
"j => о",
"k => л",
"l => д",
"m => ь",
"n => т",
"o => щ",
"p => з",
"r => к",
"s => ы",
"t => е",
"u => г",
"v => м",
"w => ц",
"x => ч",
"y => н",
"z => я",
"A => Ф",
"B => И",
"C => С",
"D => В",
"E => У",
"F => А",
"G => П",
"H => Р",
"I => Ш",
"J => О",
"K => Л",
"L => Д",
"M => Ь",
"N => Т",
"O => Щ",
"P => З",
"R => К",
"S => Ы",
"T => Е",
"U => Г",
"V => М",
"W => Ц",
"X => Ч",
"Y => Н",
"Z => Я",
"[ => х",
"] => ъ",
"; => ж",
"< => б",
"> => ю"
];
}
24 changes: 19 additions & 5 deletions src/Sitko.Core.Search.OpenSearch/OpenSearchSearcher.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ public class OpenSearchSearcher<TSearchModel>(
private OpenSearchModuleOptions Options => optionsMonitor.CurrentValue;
private OpenSearchClient? client;
private const string CustomAnalyze = "custom_analyze";
private const string CustomCharFilterAnalyze = "char_filter_analyze";
private const string StemmerName = "custom_stemmer";
private const string CustomCharFilter = "rus_en_key";

public async Task<bool> AddOrUpdateAsync(string indexName, IEnumerable<TSearchModel> searchModels,
CancellationToken cancellationToken = default)
Expand Down Expand Up @@ -264,12 +266,20 @@ private static SearchDescriptor<TSearchModel> GetSearchRequest(SearchDescriptor<
}

private AnalysisDescriptor CreateAnalysisDescriptor(AnalysisDescriptor a) =>
a.Analyzers(aa => aa.Custom(CustomAnalyze, ca => ca
.Tokenizer("standard")
.Filters("lowercase", "stop", "snowball", StemmerName)
a.Analyzers(aa =>
aa.Custom(CustomAnalyze, ca => ca
.Tokenizer("standard")
.Filters("lowercase", "stop", "snowball", StemmerName))
.Custom(CustomCharFilterAnalyze, ca => ca
.Tokenizer("standard")
.Filters("lowercase", "stop")
.CharFilters(CustomCharFilter))
)
).TokenFilters(descriptor =>
descriptor.Stemmer(StemmerName, filterDescriptor => filterDescriptor.Language(Options.CustomStemmer)));
.CharFilters(descriptor =>
descriptor.Mapping(CustomCharFilter,
filterDescriptor => filterDescriptor.Mappings(OpenSearchHelper.RusEnKeys)))
.TokenFilters(descriptor =>
descriptor.Stemmer(StemmerName, filterDescriptor => filterDescriptor.Language(Options.CustomStemmer)));

private CreateIndexDescriptor CreateIndexDescriptor(CreateIndexDescriptor createIndexDescriptor) =>
createIndexDescriptor.Settings(s => s.Analysis(CreateAnalysisDescriptor))
Expand All @@ -279,6 +289,10 @@ private CreateIndexDescriptor CreateIndexDescriptor(CreateIndexDescriptor create
.Name(n => n.Content)
.Analyzer(CustomAnalyze)
)
.Text(t => t
.Name(n => n.Title)
.Analyzer(CustomCharFilterAnalyze)
)
)
);
}
20 changes: 20 additions & 0 deletions tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,26 @@ public async Task SearchByNumbersTestAsync(int foundDocs, string searchText, Sea
var result2 = await searchProvider.SearchAsync(searchText, 10, searchType);
result2.Length.Should().Be(foundDocs);
}

[Fact]
public async Task IncorrectLayoutKeyboardTestAsync()
{
var scope = await GetScopeAsync();
var provider = scope.GetService<TestModelProvider>();
var searchProvider = scope.GetService<ISearchProvider<TestModel, Guid>>();
await searchProvider.DeleteIndexAsync();
await searchProvider.InitAsync();

var firstModel = new TestModel { Title = "kolesa", Description = "MMI", Url = "/page/" };
var secondModel = new TestModel { Title = "MMI", Description = "mmicentre", Url = "mmicentre" };
provider.AddModel(firstModel).AddModel(secondModel);

await searchProvider.AddOrUpdateEntitiesAsync(provider.Models.ToArray());
await Task.Delay(TimeSpan.FromSeconds(5));

var result = await searchProvider.SearchAsync("лщдуыф", 10, SearchType.Wildcard);
result.Length.Should().Be(1);
}
}

public class OpenSearchTestScope : BaseTestScope
Expand Down

0 comments on commit 92eb6fc

Please sign in to comment.