Tentative on fixing not filter on elasticsearch.

microsoft · Jul 16, 2024 · ae85a29 · ae85a29
1 parent eade9dc
commit ae85a29
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 42 deletions.
diff --git a/extensions/Elasticsearch/Elasticsearch.FunctionalTests/Additional/IndexNameTests.cs b/extensions/Elasticsearch/Elasticsearch.FunctionalTests/Additional/IndexNameTests.cs
@@ -71,7 +71,7 @@ public void BadIndexNamesAreRejected(string indexName, int errorCount)
             $"" +
             $"The expected number of errors was {errorCount}.");
 
-        Assert.True(errorCount == exception.Errors.Count(), $"The number of errprs expected is different than the number of errors found.");
+        Assert.True(errorCount == exception.Errors.Count(), $"The number of errors expected is different than the number of errors found.");
     }
 
     [Fact]

diff --git a/extensions/Elasticsearch/Elasticsearch.FunctionalTests/Additional/KernelMemoryTests.cs b/extensions/Elasticsearch/Elasticsearch.FunctionalTests/Additional/KernelMemoryTests.cs
@@ -15,12 +15,29 @@ public class KernelMemoryTests : MemoryDbFunctionalTest
     public KernelMemoryTests(IConfiguration cfg, ITestOutputHelper output)
         : base(cfg, output)
     {
-        this.KernelMemory = new KernelMemoryBuilder()
-            .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" })
-            .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound })
-            .WithOpenAI(this.OpenAiConfig)
-            .WithElasticsearchMemoryDb(this.ElasticsearchConfig)
-            .Build<MemoryServerless>();
+        if (cfg.GetValue<bool>("UseAzureOpenAI"))
+        {
+            Assert.False(string.IsNullOrEmpty(this.AzureOpenAIEmbeddingConfiguration.APIKey));
+
+            this.KernelMemory = new KernelMemoryBuilder()
+                .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" })
+                .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound })
+                .WithAzureOpenAITextGeneration(this.AzureOpenAITextConfiguration)
+                .WithAzureOpenAITextEmbeddingGeneration(this.AzureOpenAIEmbeddingConfiguration)
+                .WithElasticsearchMemoryDb(this.ElasticsearchConfig)
+                .Build<MemoryServerless>();
+        }
+        else
+        {
+            Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey));
+
+            this.KernelMemory = new KernelMemoryBuilder()
+                .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" })
+                .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound })
+                .WithOpenAI(this.OpenAiConfig)
+                .WithElasticsearchMemoryDb(this.ElasticsearchConfig)
+                .Build<MemoryServerless>();
+        }
     }
 
     public IKernelMemory KernelMemory { get; }

diff --git a/extensions/Elasticsearch/Elasticsearch.FunctionalTests/Additional/MemoryDbFunctionalTest.cs b/extensions/Elasticsearch/Elasticsearch.FunctionalTests/Additional/MemoryDbFunctionalTest.cs
@@ -2,6 +2,7 @@
 
 using Elastic.Clients.Elasticsearch;
 using Microsoft.KernelMemory.AI;
+using Microsoft.KernelMemory.AI.AzureOpenAI;
 using Microsoft.KernelMemory.AI.OpenAI;
 using Microsoft.KernelMemory.MemoryDb.Elasticsearch;
 using Microsoft.KernelMemory.MemoryDb.Elasticsearch.Internals;
@@ -24,12 +25,21 @@ protected MemoryDbFunctionalTest(IConfiguration cfg, ITestOutputHelper output)
         : base(cfg, output)
     {
         this.Output = output ?? throw new ArgumentNullException(nameof(output));
-
+        if (cfg.GetValue<bool>("UseAzureOpenAI"))
+        {
+            this.TextEmbeddingGenerator = new AzureOpenAITextEmbeddingGenerator(
+                config: base.AzureOpenAIEmbeddingConfiguration,
+                textTokenizer: default,
+                loggerFactory: default);
+        }
+        else
+        {
 #pragma warning disable KMEXP01 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
-        this.TextEmbeddingGenerator = new OpenAITextEmbeddingGenerator(
-            config: base.OpenAiConfig,
-            textTokenizer: default,
-            loggerFactory: default);
+            this.TextEmbeddingGenerator = new OpenAITextEmbeddingGenerator(
+                config: base.OpenAiConfig,
+                textTokenizer: default,
+                loggerFactory: default);
+        }
 #pragma warning restore KMEXP01 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
 
         this.Client = new ElasticsearchClient(base.ElasticsearchConfig.ToElasticsearchClientSettings());

diff --git a/extensions/Elasticsearch/Elasticsearch.FunctionalTests/DefaultTests.cs b/extensions/Elasticsearch/Elasticsearch.FunctionalTests/DefaultTests.cs
@@ -15,18 +15,31 @@ public class DefaultTests : BaseFunctionalTestCase
 
     public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, output)
     {
-        Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey));
-
         this._esConfig = cfg.GetSection("KernelMemory:Services:Elasticsearch").Get<ElasticsearchConfig>()!;
 
-        this._memory = new KernelMemoryBuilder()
-            .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" })
-            .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound })
-            .WithOpenAI(this.OpenAiConfig)
-            // .WithAzureOpenAITextGeneration(this.AzureOpenAITextConfiguration)
-            // .WithAzureOpenAITextEmbeddingGeneration(this.AzureOpenAIEmbeddingConfiguration)
-            .WithElasticsearchMemoryDb(this._esConfig)
-            .Build<MemoryServerless>();
+        if (cfg.GetValue<bool>("UseAzureOpenAI"))
+        {
+            Assert.False(string.IsNullOrEmpty(this.AzureOpenAIEmbeddingConfiguration.APIKey));
+
+            this._memory = new KernelMemoryBuilder()
+                .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" })
+                .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound })
+                .WithAzureOpenAITextGeneration(this.AzureOpenAITextConfiguration)
+                .WithAzureOpenAITextEmbeddingGeneration(this.AzureOpenAIEmbeddingConfiguration)
+                .WithElasticsearchMemoryDb(this._esConfig)
+                .Build<MemoryServerless>();
+        }
+        else
+        {
+            Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey));
+
+            this._memory = new KernelMemoryBuilder()
+                .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" })
+                .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound })
+                .WithOpenAI(this.OpenAiConfig)
+                .WithElasticsearchMemoryDb(this._esConfig)
+                .Build<MemoryServerless>();
+        }
     }
 
     [Fact]

diff --git a/extensions/Elasticsearch/Elasticsearch.FunctionalTests/Elasticsearch.FunctionalTests.csproj b/extensions/Elasticsearch/Elasticsearch.FunctionalTests/Elasticsearch.FunctionalTests.csproj
@@ -9,6 +9,7 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <IsPackable>false</IsPackable>
+        <UserSecretsId>3e71ea16-d53c-41a5-9202-c3f264cf1c24</UserSecretsId>
     </PropertyGroup>
 
     <ItemGroup>

diff --git a/extensions/Elasticsearch/Elasticsearch/ElasticsearchMemory.cs b/extensions/Elasticsearch/Elasticsearch/ElasticsearchMemory.cs
@@ -306,40 +306,55 @@ private QueryDescriptor<ElasticsearchMemoryRecord> ConvertTagFilters(
         QueryDescriptor<ElasticsearchMemoryRecord> qd,
         ICollection<MemoryFilter>? filters = null)
     {
-        if ((filters == null) || (filters.Count == 0))
-        {
-            qd.MatchAll();
-            return qd;
-        }
-
-        filters = filters.Where(f => f.Keys.Count > 0)
-            .ToList(); // Remove empty filters
-
-        if (filters.Count == 0)
+        var hasOneNotEmptyFilter = filters != null && filters.Any(f => !f.IsEmpty());
+        if (!hasOneNotEmptyFilter)
         {
             qd.MatchAll();
             return qd;
         }
 
         List<Query> super = new();
 
-        foreach (MemoryFilter filter in filters)
+        foreach (MemoryFilter filter in filters!)
         {
             List<Query> thisMust = new();
 
             // Each filter is a list of key/value pairs.
-            foreach (var pair in filter.Pairs)
+            foreach (var baseFilter in filter.GetAllFilters())
             {
-                Query newTagQuery = new TermQuery(ElasticsearchMemoryRecord.TagsName) { Value = pair.Key };
-                Query termQuery = new TermQuery(ElasticsearchMemoryRecord.TagsValue) { Value = pair.Value ?? string.Empty };
+                if (baseFilter is EqualFilter eq)
+                {
+                    Query newTagQuery = new TermQuery(ElasticsearchMemoryRecord.TagsName) { Value = eq.Key };
+                    Query termQuery = new TermQuery(ElasticsearchMemoryRecord.TagsValue) { Value = eq.Value ?? string.Empty };
+
+                    newTagQuery &= termQuery;
+
+                    var nestedQd = new NestedQuery();
+                    nestedQd.Path = ElasticsearchMemoryRecord.TagsField;
+                    nestedQd.Query = newTagQuery;
 
-                newTagQuery &= termQuery;
+                    thisMust.Add(nestedQd);
+                }
+                else if (baseFilter is NotEqualFilter neq)
+                {
+                    Query newTagQuery = new TermQuery(ElasticsearchMemoryRecord.TagsName) { Value = neq.Key };
+                    Query termQuery = new TermQuery(ElasticsearchMemoryRecord.TagsValue) { Value = neq.Value ?? string.Empty };
+
+                    newTagQuery &= termQuery;
 
-                var nestedQd = new NestedQuery();
-                nestedQd.Path = ElasticsearchMemoryRecord.TagsField;
-                nestedQd.Query = newTagQuery;
+                    var nestedQd = new NestedQuery();
+                    nestedQd.Path = ElasticsearchMemoryRecord.TagsField;
+                    nestedQd.Query = new BoolQuery
+                    {
+                        MustNot = new List<Query> { newTagQuery }
+                    };
 
-                thisMust.Add(nestedQd);
+                    thisMust.Add(nestedQd);
+                }
+                else
+                {
+                    throw new ElasticsearchException($"Filter type {baseFilter.GetType().Name} is not supported.");
+                }
             }
 
             var filterQuery = new BoolQuery();

diff --git a/extensions/Postgres/Postgres/PostgresMemory.cs b/extensions/Postgres/Postgres/PostgresMemory.cs
@@ -281,7 +281,7 @@ private static string NormalizeTableNamePrefix(string? name)
                 .Select(x => $"{x.Key}{Constants.ReservedEqualsChar}{x.Value}").ToList();
             List<string> safeSqlPlaceholders = new();
 
-            List<string> conditions = new ();
+            List<string> conditions = new();
 
             if (equalTags.Count > 0)
             {