Bugfix/Missing Filter for VectorStore to Document (FlowiseAI#2285)

add filter for vector store to document
menloparklab · Apr 29, 2024 · 2b1273c · 2b1273c
1 parent c4eb75d
commit 2b1273c
Show file tree

Hide file tree

Showing 17 changed files with 67 additions and 9 deletions.
diff --git a/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts b/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts
@@ -70,12 +70,19 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
         const output = nodeData.outputs?.output as string
 
         const topK = (vectorStore as any)?.k ?? 4
+        const _filter = (vectorStore as any)?.filter
 
-        const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK)
+        // If it is already pre-defined in lc_kwargs, then don't pass it again
+        const filter = vectorStore.lc_kwargs.filter ? undefined : _filter
+        if (vectorStore.lc_kwargs.filter) {
+            ;(vectorStore as any).filter = vectorStore.lc_kwargs.filter
+        }
+
+        const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK, filter)
         // eslint-disable-next-line no-console
         console.log('\x1b[94m\x1b[1m\n*****VectorStore Documents*****\n\x1b[0m\x1b[0m')
         // eslint-disable-next-line no-console
-        console.log(docs)
+        console.log(JSON.stringify(docs, null, 2))
 
         if (output === 'document') {
             let finaldocs = []

diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma.ts b/packages/components/nodes/vectorstores/Chroma/Chroma.ts
@@ -187,6 +187,9 @@ class Chroma_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (chromaMetadataFilter) {
+                ;(vectorStore as any).filter = obj.filter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts
@@ -117,6 +117,9 @@ class Chroma_Existing_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (chromaMetadataFilter) {
+                ;(vectorStore as any).filter = obj.filter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/Milvus/Milvus.ts b/packages/components/nodes/vectorstores/Milvus/Milvus.ts
@@ -199,6 +199,9 @@ class Milvus_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (milvusFilter) {
+                ;(vectorStore as any).filter = milvusFilter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/Milvus/Milvus_Existing.ts b/packages/components/nodes/vectorstores/Milvus/Milvus_Existing.ts
@@ -189,6 +189,9 @@ class Milvus_Existing_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (milvusFilter) {
+                ;(vectorStore as any).filter = milvusFilter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts
@@ -190,7 +190,7 @@ class Pinecone_VectorStores implements INode {
 
         const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)
 
-        return resolveVectorStoreOrRetriever(nodeData, vectorStore)
+        return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter)
     }
 }
 

diff --git a/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts b/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts
@@ -354,6 +354,9 @@ class Qdrant_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (queryFilter) {
+                ;(vectorStore as any).filter = retrieverConfig.filter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/Qdrant/Qdrant_Existing.ts b/packages/components/nodes/vectorstores/Qdrant/Qdrant_Existing.ts
@@ -182,6 +182,9 @@ class Qdrant_Existing_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (queryFilter) {
+                ;(vectorStore as any).filter = retrieverConfig.filter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/Supabase/Supabase.ts b/packages/components/nodes/vectorstores/Supabase/Supabase.ts
@@ -186,7 +186,7 @@ class Supabase_VectorStores implements INode {
 
         const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, obj)
 
-        return resolveVectorStoreOrRetriever(nodeData, vectorStore)
+        return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter)
     }
 }
 

diff --git a/packages/components/nodes/vectorstores/Supabase/Supabase_Exisiting.ts b/packages/components/nodes/vectorstores/Supabase/Supabase_Exisiting.ts
@@ -119,6 +119,9 @@ class Supabase_Existing_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (supabaseMetadataFilter) {
+                ;(vectorStore as any).filter = obj.filter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/Vectara/Vectara.ts b/packages/components/nodes/vectorstores/Vectara/Vectara.ts
@@ -272,6 +272,9 @@ class Vectara_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (vectaraMetadataFilter) {
+                ;(vectorStore as any).filter = vectaraFilter.filter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/Vectara/Vectara_Existing.ts b/packages/components/nodes/vectorstores/Vectara/Vectara_Existing.ts
@@ -127,6 +127,9 @@ class VectaraExisting_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (vectaraMetadataFilter) {
+                ;(vectorStore as any).filter = vectaraFilter.filter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/VectorStoreUtils.ts b/packages/components/nodes/vectorstores/VectorStoreUtils.ts
@@ -1,11 +1,19 @@
 import { INodeData } from '../../src'
+import { VectorStore } from '@langchain/core/vectorstores'
 
-export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore: any) => {
+export const resolveVectorStoreOrRetriever = (
+    nodeData: INodeData,
+    vectorStore: VectorStore,
+    metadataFilter?: string | object | undefined
+) => {
     const output = nodeData.outputs?.output as string
     const searchType = nodeData.outputs?.searchType as string
     const topK = nodeData.inputs?.topK as string
     const k = topK ? parseFloat(topK) : 4
 
+    // If it is already pre-defined in lc_kwargs, then don't pass it again
+    const filter = vectorStore?.lc_kwargs?.filter ? undefined : metadataFilter
+
     if (output === 'retriever') {
         if ('mmr' === searchType) {
             const fetchK = nodeData.inputs?.fetchK as string
@@ -15,17 +23,19 @@ export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore:
             return vectorStore.asRetriever({
                 searchType: 'mmr',
                 k: k,
+                filter,
                 searchKwargs: {
                     fetchK: f,
                     lambda: l
                 }
             })
         } else {
             // "searchType" is "similarity"
-            return vectorStore.asRetriever(k)
+            return vectorStore.asRetriever(k, filter)
         }
     } else if (output === 'vectorStore') {
         ;(vectorStore as any).k = k
+        ;(vectorStore as any).filter = filter
         return vectorStore
     }
 }

diff --git a/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts b/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts
@@ -114,6 +114,13 @@ class Weaviate_VectorStores implements INode {
                 type: 'number',
                 additionalParams: true,
                 optional: true
+            },
+            {
+                label: 'Weaviate Search Filter',
+                name: 'weaviateFilter',
+                type: 'json',
+                additionalParams: true,
+                optional: true
             }
         ]
         addMMRInputParams(this.inputs)
@@ -203,6 +210,7 @@ class Weaviate_VectorStores implements INode {
         const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string
         const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
         const embeddings = nodeData.inputs?.embeddings as Embeddings
+        let weaviateFilter = nodeData.inputs?.weaviateFilter
 
         const credentialData = await getCredentialData(nodeData.credential ?? '', options)
         const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
@@ -223,10 +231,13 @@ class Weaviate_VectorStores implements INode {
 
         if (weaviateTextKey) obj.textKey = weaviateTextKey
         if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
+        if (weaviateFilter) {
+            weaviateFilter = typeof weaviateFilter === 'object' ? weaviateFilter : JSON.parse(weaviateFilter)
+        }
 
         const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj)
 
-        return resolveVectorStoreOrRetriever(nodeData, vectorStore)
+        return resolveVectorStoreOrRetriever(nodeData, vectorStore, weaviateFilter)
     }
 }
 

diff --git a/packages/components/nodes/vectorstores/Zep/Zep.ts b/packages/components/nodes/vectorstores/Zep/Zep.ts
@@ -165,7 +165,7 @@ class Zep_VectorStores implements INode {
 
         const vectorStore = await ZepExistingVS.fromExistingIndex(embeddings, zepConfig)
 
-        return resolveVectorStoreOrRetriever(nodeData, vectorStore)
+        return resolveVectorStoreOrRetriever(nodeData, vectorStore, zepConfig.filter)
     }
 }
 

diff --git a/packages/components/nodes/vectorstores/Zep/Zep_Existing.ts b/packages/components/nodes/vectorstores/Zep/Zep_Existing.ts
@@ -125,6 +125,9 @@ class Zep_Existing_VectorStores implements INode {
             return retriever
         } else if (output === 'vectorStore') {
             ;(vectorStore as any).k = k
+            if (zepMetadataFilter) {
+                ;(vectorStore as any).filter = zepConfig.filter
+            }
             return vectorStore
         }
         return vectorStore

diff --git a/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts b/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts
@@ -131,7 +131,7 @@ class Zep_CloudVectorStores implements INode {
         }
         zepConfig.client = await ZepClient.init(zepConfig.apiKey)
         const vectorStore = await ZepExistingVS.init(zepConfig)
-        return resolveVectorStoreOrRetriever(nodeData, vectorStore)
+        return resolveVectorStoreOrRetriever(nodeData, vectorStore, zepConfig.filter)
     }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -190,7 +190,7 @@ class Pinecone_VectorStores implements INode { @@
             const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)
-            return resolveVectorStoreOrRetriever(nodeData, vectorStore)
+            return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter)
         }
     }
@@ Expand Down @@