From d46dded0f74b03ba9aa9fd01334d88a29aa55bda Mon Sep 17 00:00:00 2001 From: IgorAlymov Date: Mon, 29 Jul 2024 13:42:24 +0500 Subject: [PATCH 1/2] feat: added Wildcard searching Refs: SITKO-CORE-T-20 --- .../ElasticSearcher.cs | 2 +- .../OpenSearchSearcher.cs | 20 +++++-- src/Sitko.Core.Search/BaseSearchProvider.cs | 8 +-- src/Sitko.Core.Search/ISearchProvider.cs | 4 +- src/Sitko.Core.Search/ISearcher.cs | 2 +- src/Sitko.Core.Search/SearchType.cs | 7 +++ .../ElasticSearchTests.cs | 2 +- .../OpenSearchTests.cs | 60 ++++++++++++++++++- 8 files changed, 88 insertions(+), 17 deletions(-) create mode 100644 src/Sitko.Core.Search/SearchType.cs diff --git a/src/Sitko.Core.Search.ElasticSearch/ElasticSearcher.cs b/src/Sitko.Core.Search.ElasticSearch/ElasticSearcher.cs index 111709285..19de610b5 100644 --- a/src/Sitko.Core.Search.ElasticSearch/ElasticSearcher.cs +++ b/src/Sitko.Core.Search.ElasticSearch/ElasticSearcher.cs @@ -101,7 +101,7 @@ public async Task CountAsync(string indexName, string term, CancellationTo return resultsCount.Count; } - public async Task SearchAsync(string indexName, string term, int limit, + public async Task SearchAsync(string indexName, string term, int limit, SearchType searchType, CancellationToken cancellationToken = default) { indexName = $"{Options.Prefix}_{indexName}"; diff --git a/src/Sitko.Core.Search.OpenSearch/OpenSearchSearcher.cs b/src/Sitko.Core.Search.OpenSearch/OpenSearchSearcher.cs index f96aae966..2a9dc5231 100644 --- a/src/Sitko.Core.Search.OpenSearch/OpenSearchSearcher.cs +++ b/src/Sitko.Core.Search.OpenSearch/OpenSearchSearcher.cs @@ -98,11 +98,11 @@ public async Task CountAsync(string indexName, string term, CancellationTo } public async Task SearchAsync(string indexName, string term, int limit, - CancellationToken cancellationToken = default) + SearchType searchType, CancellationToken cancellationToken = default) { indexName = $"{Options.Prefix}_{indexName}"; var results = await GetClient() - .SearchAsync(x => GetSearchRequest(x, indexName, term, limit), cancellationToken); + .SearchAsync(x => GetSearchRequest(x, indexName, term, searchType, limit), cancellationToken); if (results.ServerError != null) { logger.LogError("Error while searching in {IndexName}: {ErrorText}", indexName, results.ServerError); @@ -240,11 +240,21 @@ private static string GetSearchText(string? term) } private static SearchDescriptor GetSearchRequest(SearchDescriptor descriptor, - string indexName, string term, int limit = 0) + string indexName, string term, SearchType searchType, int limit = 0) { var names = GetSearchText(term); - return descriptor.Query(q => q.QueryString(qs => qs.Query(names))) - .Sort(s => s.Descending(SortSpecialField.Score).Descending(model => model.Date)) + switch (searchType) + { + case SearchType.Morphology: + descriptor.Query(q => q.QueryString(qs => qs.Query(names))); + break; + case SearchType.Wildcard: + descriptor.Query(q => + q.QueryString(qs => qs.Query($"*{names}*").AnalyzeWildcard())); + break; + } + + return descriptor.Sort(s => s.Descending(SortSpecialField.Score).Descending(model => model.Date)) .Size(limit > 0 ? limit : 20) .Index(indexName.ToLowerInvariant()); } diff --git a/src/Sitko.Core.Search/BaseSearchProvider.cs b/src/Sitko.Core.Search/BaseSearchProvider.cs index 14946ff53..9fc0b5e63 100644 --- a/src/Sitko.Core.Search/BaseSearchProvider.cs +++ b/src/Sitko.Core.Search/BaseSearchProvider.cs @@ -31,16 +31,16 @@ public Task CountAsync(string term, CancellationToken cancellationToken = public Task InitAsync(CancellationToken cancellationToken = default) => searcher.InitAsync(IndexName, cancellationToken); - public async Task SearchAsync(string term, int limit, CancellationToken cancellationToken = default) + public async Task SearchAsync(string term, int limit, SearchType searchType, CancellationToken cancellationToken = default) { - var result = await searcher.SearchAsync(IndexName, term, limit, cancellationToken); + var result = await searcher.SearchAsync(IndexName, term, limit, searchType, cancellationToken); return await LoadEntities(result, cancellationToken); } - public async Task GetIdsAsync(string term, int limit, + public async Task GetIdsAsync(string term, int limit, SearchType searchType, CancellationToken cancellationToken = default) { - var result = await searcher.SearchAsync(IndexName, term, limit, cancellationToken); + var result = await searcher.SearchAsync(IndexName, term, limit, searchType, cancellationToken); return result.Select(m => ParseId(m.Id)).ToArray(); } diff --git a/src/Sitko.Core.Search/ISearchProvider.cs b/src/Sitko.Core.Search/ISearchProvider.cs index 8b2c14a8b..337b840fc 100644 --- a/src/Sitko.Core.Search/ISearchProvider.cs +++ b/src/Sitko.Core.Search/ISearchProvider.cs @@ -10,8 +10,8 @@ public interface ISearchProvider public interface ISearchProvider : ISearchProvider where TEntity : class { - Task SearchAsync(string term, int limit, CancellationToken cancellationToken = default); - Task GetIdsAsync(string term, int limit, CancellationToken cancellationToken = default); + Task SearchAsync(string term, int limit, SearchType searchType, CancellationToken cancellationToken = default); + Task GetIdsAsync(string term, int limit, SearchType searchType, CancellationToken cancellationToken = default); Task GetSimilarAsync(string id, int limit, CancellationToken cancellationToken = default); Task GetSimilarIdsAsync(string id, int limit, diff --git a/src/Sitko.Core.Search/ISearcher.cs b/src/Sitko.Core.Search/ISearcher.cs index ed8221548..92c6e1c6c 100644 --- a/src/Sitko.Core.Search/ISearcher.cs +++ b/src/Sitko.Core.Search/ISearcher.cs @@ -10,7 +10,7 @@ Task DeleteAsync(string indexName, IEnumerable searchModels, Task DeleteAsync(string indexName, CancellationToken cancellationToken = default); Task CountAsync(string indexName, string term, CancellationToken cancellationToken = default); - Task SearchAsync(string indexName, string term, int limit, CancellationToken cancellationToken = default); + Task SearchAsync(string indexName, string term, int limit, SearchType searchType, CancellationToken cancellationToken = default); Task GetSimilarAsync(string indexName, string id, int limit, CancellationToken cancellationToken = default); diff --git a/src/Sitko.Core.Search/SearchType.cs b/src/Sitko.Core.Search/SearchType.cs new file mode 100644 index 000000000..74c28a1bf --- /dev/null +++ b/src/Sitko.Core.Search/SearchType.cs @@ -0,0 +1,7 @@ +namespace Sitko.Core.Search; + +public enum SearchType +{ + Wildcard = 0, + Morphology = 1 +} diff --git a/tests/Sitko.Core.Search.ElasticSearch.Tests/ElasticSearchTests.cs b/tests/Sitko.Core.Search.ElasticSearch.Tests/ElasticSearchTests.cs index 4e865ca0a..74f1fd0c3 100644 --- a/tests/Sitko.Core.Search.ElasticSearch.Tests/ElasticSearchTests.cs +++ b/tests/Sitko.Core.Search.ElasticSearch.Tests/ElasticSearchTests.cs @@ -43,7 +43,7 @@ public async Task Search() await searchProvider.AddOrUpdateEntitiesAsync(provider.Models.ToArray()); await Task.Delay(TimeSpan.FromSeconds(5)); - var result = await searchProvider.SearchAsync("samsung", 10); + var result = await searchProvider.SearchAsync("samsung", 10, SearchType.Morphology); Assert.Equal(provider.Models.Count, result.Length); Assert.Equal(barModel.Id, result.First().Id); } diff --git a/tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs b/tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs index f9294bdea..8f12e7c71 100644 --- a/tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs +++ b/tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs @@ -38,7 +38,7 @@ public async Task SearchAsync() await searchProvider.AddOrUpdateEntitiesAsync(provider.Models.ToArray()); await Task.Delay(TimeSpan.FromSeconds(5)); - var result = await searchProvider.SearchAsync("samsung", 10); + var result = await searchProvider.SearchAsync("samsung", 10, SearchType.Morphology); result.Length.Should().Be(provider.Models.Count); result.First().Id.Should().Be(barModel.Id); } @@ -69,7 +69,7 @@ public async Task MorphologyRusTestAsync(int foundDocs, string searchText) await searchProvider.AddOrUpdateEntitiesAsync(provider.Models.ToArray()); await Task.Delay(TimeSpan.FromSeconds(5)); - var result = await searchProvider.SearchAsync(searchText, 10); + var result = await searchProvider.SearchAsync(searchText, 10, SearchType.Morphology); result.Length.Should().Be(foundDocs); } @@ -92,9 +92,63 @@ public async Task MorphologyEngTestAsync() await searchProvider.AddOrUpdateEntitiesAsync(provider.Models.ToArray()); await Task.Delay(TimeSpan.FromSeconds(5)); - var result = await searchProvider.SearchAsync("walked", 10); + var result = await searchProvider.SearchAsync("walked", 10, SearchType.Morphology); result.Length.Should().Be(3); } + + [Theory(DisplayName = "PartialSearchEngTest")] + [InlineData(1, "74")] + [InlineData(1, "kol")] + [InlineData(1, "kolesa")] + [InlineData(1, "74ko")] + public async Task PartialSearchEngTestAsync(int foundDocs, string searchText) + { + var scope = await GetScopeAsync(); + var provider = scope.GetService(); + var searchProvider = scope.GetService>(); + await searchProvider.DeleteIndexAsync(); + await searchProvider.InitAsync(); + + var firstModel = new TestModel { Title = "MMI", Description = "74kolesa", Url = "mmicentre" }; + var secondModel = new TestModel { Title = "MMI", Description = "walked", Url = "mmicentre" }; + var thirdModel = + new TestModel { Title = "MMI", Description = "walking", Url = "mmicentre" }; + var forthModel = new TestModel { Title = "MMI", Description = "MMI", Url = "mmicentre" }; + provider.AddModel(firstModel).AddModel(secondModel).AddModel(thirdModel).AddModel(forthModel); + + await searchProvider.AddOrUpdateEntitiesAsync(provider.Models.ToArray()); + await Task.Delay(TimeSpan.FromSeconds(5)); + + var result = await searchProvider.SearchAsync(searchText, 10, SearchType.Wildcard); + result.Length.Should().Be(foundDocs); + } + + [Theory(DisplayName = "PartialSearchRusTest")] + [InlineData(1, "74")] + [InlineData(1, "кол")] + [InlineData(1, "колеса")] + [InlineData(1, "74ко")] + public async Task PartialSearchRusTestAsync(int foundDocs, string searchText) + { + var scope = await GetScopeAsync(); + var provider = scope.GetService(); + var searchProvider = scope.GetService>(); + await searchProvider.DeleteIndexAsync(); + await searchProvider.InitAsync(); + + var firstModel = new TestModel { Title = "MMI", Description = "74колеса", Url = "mmicentre" }; + var secondModel = new TestModel { Title = "MMI", Description = "walked", Url = "mmicentre" }; + var thirdModel = + new TestModel { Title = "MMI", Description = "walking", Url = "mmicentre" }; + var forthModel = new TestModel { Title = "MMI", Description = "MMI", Url = "mmicentre" }; + provider.AddModel(firstModel).AddModel(secondModel).AddModel(thirdModel).AddModel(forthModel); + + await searchProvider.AddOrUpdateEntitiesAsync(provider.Models.ToArray()); + await Task.Delay(TimeSpan.FromSeconds(5)); + + var result = await searchProvider.SearchAsync(searchText, 10, SearchType.Wildcard); + result.Length.Should().Be(foundDocs); + } } public class OpenSearchTestScope : BaseTestScope From 89af8b265c38b531dffcf453276251380389dda8 Mon Sep 17 00:00:00 2001 From: IgorAlymov Date: Mon, 29 Jul 2024 13:59:48 +0500 Subject: [PATCH 2/2] fix: fixed PartialSearchRusTest Refs: SITKO-CORE-T-20 --- tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs b/tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs index 8f12e7c71..534dbcd37 100644 --- a/tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs +++ b/tests/Sitko.Core.Search.OpenSearch.Tests/OpenSearchTests.cs @@ -124,9 +124,9 @@ public async Task PartialSearchEngTestAsync(int foundDocs, string searchText) } [Theory(DisplayName = "PartialSearchRusTest")] - [InlineData(1, "74")] [InlineData(1, "кол")] - [InlineData(1, "колеса")] + [InlineData(1, "74кол")] + [InlineData(1, "74колес")] [InlineData(1, "74ко")] public async Task PartialSearchRusTestAsync(int foundDocs, string searchText) {