Update BM25 retreiver to use metadata

run-llama · Sep 27, 2024 · 43bc7e7 · 43bc7e7
1 parent a719cee
commit 43bc7e7
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/...x-integrations/retrievers/llama-index-retrievers-bm25/llama_index/retrievers/bm25/base.py b/...x-integrations/retrievers/llama-index-retrievers-bm25/llama_index/retrievers/bm25/base.py
@@ -8,7 +8,13 @@
 from llama_index.core.callbacks.base import CallbackManager
 from llama_index.core.constants import DEFAULT_SIMILARITY_TOP_K
 from llama_index.core.indices.vector_store.base import VectorStoreIndex
-from llama_index.core.schema import BaseNode, IndexNode, NodeWithScore, QueryBundle
+from llama_index.core.schema import (
+    BaseNode,
+    IndexNode,
+    NodeWithScore,
+    QueryBundle,
+    MetadataMode,
+)
 from llama_index.core.storage.docstore.types import BaseDocumentStore
 from llama_index.core.vector_stores.utils import (
     node_to_metadata_dict,
@@ -75,7 +81,7 @@ def __init__(
             self.corpus = [node_to_metadata_dict(node) for node in nodes]
 
             corpus_tokens = bm25s.tokenize(
-                [node.get_content() for node in nodes],
+                [node.get_content(metadata_mode=MetadataMode.EMBED) for node in nodes],
                 stopwords=language,
                 stemmer=self.stemmer,
                 show_progress=verbose,

diff --git a/llama-index-integrations/retrievers/llama-index-retrievers-bm25/pyproject.toml b/llama-index-integrations/retrievers/llama-index-retrievers-bm25/pyproject.toml
@@ -27,7 +27,7 @@ exclude = ["**/BUILD"]
 license = "MIT"
 name = "llama-index-retrievers-bm25"
 readme = "README.md"
-version = "0.3.0"
+version = "0.3.1"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"