From 0e7891403ddf341b6f590ae1dd99ee874fe1c6ba Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Fri, 24 May 2024 10:58:41 +0100 Subject: [PATCH 01/13] Fixup inconsistent use of v3 and v4 Weaviate clients All references to Weaviate V3 clients (weaviate.Client) have been changed to Weaviate V4 clients (weaviate.WeaviateClient). Documentation has also been modified to reflect changes. Additionally, the `auth_config` argument to `WeaviateVectorStore.from_params` has been made optional to match the underlying `weaviate.WeaviateClient` signature. --- .../vector_stores/weaviate/base.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py index 2dce3c4e12c3a..6f90fd53f2941 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py @@ -29,10 +29,10 @@ ) import weaviate -from weaviate import Client import weaviate.classes as wvc _logger = logging.getLogger(__name__) +_WEAVIATE_DEFAULT_GRPC_PORT = 50051 def _transform_weaviate_filter_condition(condition: str) -> str: @@ -98,7 +98,7 @@ class WeaviateVectorStore(BasePydanticVectorStore): k most similar nodes. Args: - weaviate_client (weaviate.Client): WeaviateClient + weaviate_client (weaviate.WeaviateClient): WeaviateClient instance from `weaviate-client` package index_name (Optional[str]): name for Weaviate classes @@ -112,7 +112,7 @@ class WeaviateVectorStore(BasePydanticVectorStore): username="", password="", ) - client = weaviate.Client( + client = weaviate.WeaviateClient( "https://llama-test-ezjahb4m.weaviate.network", auth_client_secret=resource_owner_config, ) @@ -135,7 +135,7 @@ class WeaviateVectorStore(BasePydanticVectorStore): def __init__( self, - weaviate_client: Optional[Any] = None, + weaviate_client: Optional[weaviate.WeaviateClient] = None, class_prefix: Optional[str] = None, index_name: Optional[str] = None, text_key: str = DEFAULT_TEXT_KEY, @@ -154,7 +154,7 @@ def __init__( auth_client_secret=auth_config, **client_kwargs ) else: - self._client = cast(weaviate.WeaviateClient, weaviate_client) + self._client = weaviate_client # validate class prefix starts with a capital letter if class_prefix is not None: @@ -184,7 +184,7 @@ def __init__( def from_params( cls, url: str, - auth_config: Any, + auth_config: Optional[Any] = None, index_name: Optional[str] = None, text_key: str = DEFAULT_TEXT_KEY, client_kwargs: Optional[Dict[str, Any]] = None, @@ -192,13 +192,22 @@ def from_params( ) -> "WeaviateVectorStore": """Create WeaviateVectorStore from config.""" client_kwargs = client_kwargs or {} - weaviate_client = Client( - url=url, auth_client_secret=auth_config, **client_kwargs + if isinstance(auth_config, dict): + auth_config = weaviate.auth.AuthApiKey(auth_config) + weaviate_client = weaviate.WeaviateClient( + connection_params=weaviate.connect.ConnectionParams.from_url( + url=url, + grpc_port=client_kwargs.pop("grpc_port", _WEAVIATE_DEFAULT_GRPC_PORT), + grpc_secure=client_kwargs.pop("grpc_secure", False), + ), + auth_client_secret=auth_config, + **client_kwargs, ) + weaviate_client.connect() return cls( weaviate_client=weaviate_client, url=url, - auth_config=auth_config.__dict__, + auth_config=auth_config.__dict__ if auth_config else {}, client_kwargs=client_kwargs, index_name=index_name, text_key=text_key, From faf452522c39f24fd5780c5510ff6c86ad182544 Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Fri, 24 May 2024 11:05:30 +0100 Subject: [PATCH 02/13] Bumps integration package version by patch increment to 1.0.1 --- .../llama-index-vector-stores-weaviate/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/pyproject.toml b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/pyproject.toml index 16f17c9919f0e..774d5bf26405a 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/pyproject.toml +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/pyproject.toml @@ -27,7 +27,7 @@ exclude = ["**/BUILD"] license = "MIT" name = "llama-index-vector-stores-weaviate" readme = "README.md" -version = "1.0.0" +version = "1.0.1" [tool.poetry.dependencies] python = ">=3.8.1,<4.0" From 6430ec681a1f69a111f018b4646cb2f279b55c46 Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Fri, 24 May 2024 11:27:23 +0100 Subject: [PATCH 03/13] Updates example code in WeaviateVectorStore docstring --- .../llama_index/vector_stores/weaviate/base.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py index 6f90fd53f2941..4d517d3e73114 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py @@ -106,16 +106,22 @@ class WeaviateVectorStore(BasePydanticVectorStore): `pip install llama-index-vector-stores-weaviate` ```python + from llama_index.vector_stores.weaviate import WeaviateVectorStore import weaviate - resource_owner_config = weaviate.AuthClientPassword( + resource_owner_config = weaviate.auth.AuthClientPassword( username="", password="", ) + connection_params = weaviate.connect.ConnectionParams.from_url( + url="https://llama-test-ezjahb4m.weaviate.network", + grpc_port=50051, + ) client = weaviate.WeaviateClient( - "https://llama-test-ezjahb4m.weaviate.network", + connection_params=connection_params, auth_client_secret=resource_owner_config, ) + client.connect() vector_store = WeaviateVectorStore( weaviate_client=client, index_name="LlamaIndex" From c765eeacf77bee4699b0126b1caa5dc14654ace1 Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Fri, 24 May 2024 11:28:48 +0100 Subject: [PATCH 04/13] Connects the client if not connected --- .../llama_index/vector_stores/weaviate/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py index 4d517d3e73114..9c937f340a4d0 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py @@ -121,7 +121,6 @@ class WeaviateVectorStore(BasePydanticVectorStore): connection_params=connection_params, auth_client_secret=resource_owner_config, ) - client.connect() vector_store = WeaviateVectorStore( weaviate_client=client, index_name="LlamaIndex" @@ -162,6 +161,9 @@ def __init__( else: self._client = weaviate_client + if not self._client.is_connected(): + self._client.connect() + # validate class prefix starts with a capital letter if class_prefix is not None: _logger.warning("class_prefix is deprecated, please use index_name") From df032bd801726a1d8f7e507ffcea640b01f29198 Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Fri, 24 May 2024 12:00:49 +0100 Subject: [PATCH 05/13] Reinstates required cast for _client --- .../llama_index/vector_stores/weaviate/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py index 9c937f340a4d0..af1a060b68ac7 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py @@ -159,7 +159,7 @@ def __init__( auth_client_secret=auth_config, **client_kwargs ) else: - self._client = weaviate_client + self._client = cast(weaviate.WeaviateClient, weaviate_client) if not self._client.is_connected(): self._client.connect() From dc8f15e4edf515d1ac54aa0a977b238adc949acf Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Fri, 24 May 2024 18:03:02 +0100 Subject: [PATCH 06/13] Fixes delete method which used v3 API with v4 client --- .../vector_stores/weaviate/base.py | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py index af1a060b68ac7..5500c8592a1b4 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py @@ -31,6 +31,7 @@ import weaviate import weaviate.classes as wvc + _logger = logging.getLogger(__name__) _WEAVIATE_DEFAULT_GRPC_PORT = 50051 @@ -263,29 +264,17 @@ def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None: ref_doc_id (str): The doc_id of the document to delete. """ - where_filter = { - "path": ["ref_doc_id"], - "operator": "Equal", - "valueText": ref_doc_id, - } + filters = wvc.query.Filter.by_property("ref_doc_id").equal(ref_doc_id) if "filter" in delete_kwargs and delete_kwargs["filter"] is not None: - where_filter = { - "operator": "And", - "operands": [where_filter, delete_kwargs["filter"]], # type: ignore - } - - query = ( - self._client.query.get(self.index_name) - .with_additional(["id"]) - .with_where(where_filter) - .with_limit(10000) # 10,000 is the max weaviate can fetch - ) + filters = wvc.query.Filter.all_of( + [filters, _to_weaviate_filter(delete_kwargs["filter"])] + ) + + collection = self._client.collections.get(self.index_name) + query_result = collection.query.fetch_objects(filters=filters, limit=1000) - query_result = query.do() - parsed_result = parse_get_response(query_result) - entries = parsed_result[self.index_name] - for entry in entries: - self._client.data_object.delete(entry["_additional"]["id"], self.index_name) + for entry in query_result.objects: + collection.data.delete_by_id(uuid=entry.uuid) def delete_index(self) -> None: """Delete the index associated with the client. From ee02f2a0f3220b8ead306c0a61493df65dcade13 Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Tue, 28 May 2024 10:36:12 +0100 Subject: [PATCH 07/13] Removes unrequired import --- .../llama_index/vector_stores/weaviate/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py index 5500c8592a1b4..470d6ee56e0f2 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py @@ -24,7 +24,6 @@ create_default_schema, get_all_properties, get_node_similarity, - parse_get_response, to_node, ) From 5ca80f8b94f828a0f45d20aae9745de5c9054a28 Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Tue, 28 May 2024 10:37:09 +0100 Subject: [PATCH 08/13] Makes class name method implementation less brittle --- .../llama_index/vector_stores/weaviate/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py index 470d6ee56e0f2..41b775b7fee96 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py @@ -224,7 +224,7 @@ def from_params( @classmethod def class_name(cls) -> str: - return "WeaviateVectorStore" + return cls.__name__ @property def client(self) -> Any: From 05c2d28361ec0859f7706853758e6bae2926f96c Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Tue, 28 May 2024 11:01:17 +0100 Subject: [PATCH 09/13] Updates community integration docs weaviate client create example --- docs/docs/community/integrations/vector_stores.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/docs/community/integrations/vector_stores.md b/docs/docs/community/integrations/vector_stores.md index 04f5c27984c03..da53aa77bd89d 100644 --- a/docs/docs/community/integrations/vector_stores.md +++ b/docs/docs/community/integrations/vector_stores.md @@ -702,8 +702,10 @@ resource_owner_config = weaviate.AuthClientPassword( username="", password="", ) -client = weaviate.Client( - "https://.semi.network/", +client = weaviate.WeaviateClient( + connection_params=weaviate.connect.ConnectionParams.from_url( + url="https://.semi.network/", grpc_port=50051 + ), auth_client_secret=resource_owner_config, ) From 5dc919df45b8ac9dd318a0d3e729d22539c3b4b7 Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Tue, 28 May 2024 12:57:02 +0100 Subject: [PATCH 10/13] Checks for V4 Weaviate client in client validation method --- .../llama_index/vector_stores/weaviate/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/utils.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/utils.py index 571af5e93d3e9..aa9b979bf76f2 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/utils.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/utils.py @@ -49,13 +49,15 @@ def validate_client(client: Any) -> None: """Validate client and import weaviate library.""" try: import weaviate - - client = cast(weaviate.WeaviateClient, client) except ImportError: raise ImportError( "Weaviate is not installed. " "Please install it with `pip install weaviate-client`." ) + if not isinstance(client, weaviate.WeaviateClient): + raise ValueError( + f"Invalid client type, expected weaviate.WeaviateClient, got {type(client)}" + ) cast(weaviate.WeaviateClient, client) From c6a0930024a510dd87ffb6f22fb88d1ddb2d9941 Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Tue, 28 May 2024 17:01:04 +0100 Subject: [PATCH 11/13] Reverts change to class name method for consistency with rest of code base --- .../llama_index/vector_stores/weaviate/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py index 41b775b7fee96..470d6ee56e0f2 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py @@ -224,7 +224,7 @@ def from_params( @classmethod def class_name(cls) -> str: - return cls.__name__ + return "WeaviateVectorStore" @property def client(self) -> Any: From d8f6ac8c59db5215ee2ce706b4d9660a047101fa Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Wed, 29 May 2024 10:54:40 +0100 Subject: [PATCH 12/13] Fixup incorrectly set query limit --- .../llama_index/vector_stores/weaviate/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py index 470d6ee56e0f2..d80fd3ebdbf28 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/llama_index/vector_stores/weaviate/base.py @@ -270,7 +270,10 @@ def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None: ) collection = self._client.collections.get(self.index_name) - query_result = collection.query.fetch_objects(filters=filters, limit=1000) + query_result = collection.query.fetch_objects( + filters=filters, + limit=10000, # 10,000 is the max weaviate can fetch + ) for entry in query_result.objects: collection.data.delete_by_id(uuid=entry.uuid) From 9f2086538fd95faa9e7b086b3a64cec0fad5e7e4 Mon Sep 17 00:00:00 2001 From: Chris Knight Date: Mon, 3 Jun 2024 11:29:28 +0100 Subject: [PATCH 13/13] Applies patch version bump post merge --- .../llama-index-vector-stores-weaviate/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/pyproject.toml b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/pyproject.toml index 774d5bf26405a..4a13351fcabfa 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/pyproject.toml +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-weaviate/pyproject.toml @@ -27,7 +27,7 @@ exclude = ["**/BUILD"] license = "MIT" name = "llama-index-vector-stores-weaviate" readme = "README.md" -version = "1.0.1" +version = "1.0.2" [tool.poetry.dependencies] python = ">=3.8.1,<4.0"