[Gen-Ai-Orchestrator] Add title to embedded documents for better retrieval

gen-ai/orchestrator-server/src/main/python/server/src/gen_ai_orchestrator/services/langchain/rag_chain.py

@@ -122,7 +122,9 @@ async def execute_qa_chain(query: RagQuery, debug: bool) -> RagResponse:
                         identifier=f'{doc.metadata["id"]}',
                         title=doc.metadata['title'],
                         url=doc.metadata['url'],
-                        content=doc.page_content,
+                        content=doc.metadata['original_text']
+                        if 'original_text' in doc.metadata
+                        else doc.page_content,
                     ),
                     response['source_documents'],
                 )

gen-ai/orchestrator-server/src/main/python/tock-llm-indexing-tools/index_documents.py

@@ -54,6 +54,7 @@
 
 import pandas as pd
 from docopt import docopt
+from langchain.document_loaders.dataframe import DataFrameLoader
 from langchain.embeddings.base import Embeddings
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.document_loaders import CSVLoader
@@ -100,13 +101,14 @@ def index_documents(args):
     )
 
     logging.debug(f"Read input CSV file {args['<input_csv>']}")
-    csv_loader = CSVLoader(
-        file_path=args['<input_csv>'],
-        source_column='url',
-        metadata_columns=('title', 'url'),
-        csv_args={'delimiter': '|', 'quotechar': '"'},
-    )
-    docs = csv_loader.load()
+    df = pd.read_csv(args['<input_csv>'], delimiter='|', quotechar='"')
+    # add original row nb
+    df['row'] = df.index
+    # add a 'source' metadata (this is the source's URL column at the moment,
+    # but may come from another column when input CSV file's format evolves)
+    df['source'] = df['url']
+    loader = DataFrameLoader(df, page_content_column='text')
+    docs = loader.load()
     for doc in docs:
         doc.metadata['index_session_id'] = session_uuid
         doc.metadata['index_datetime'] = formatted_datetime
@@ -120,6 +122,9 @@ def index_documents(args):
     splitted_docs = text_splitter.split_documents(docs)
     # Add chunk id ('n/N') metadata to each chunk
     splitted_docs = generate_ids_for_each_chunks(splitted_docs=splitted_docs)
+    # Add title to text (for better semantic search) and text to
+    # metadata (to easily get original text without title)
+    splitted_docs = add_title_to_text(splitted_docs=splitted_docs)
 
     logging.debug(f"Get embeddings model from {args['<embeddings_cfg>']} config file")
     with open(args['<embeddings_cfg>'], 'r') as file:
@@ -164,6 +169,21 @@ def generate_ids_for_each_chunks(
     return splitted_docs
 
 
+def add_title_to_text(
+    splitted_docs: Iterable[Document],
+) -> Iterable[Document]:
+    """Add 'title' from metadata to Document's page_content for better semantic search."""
+    for doc in splitted_docs:
+        # Store the original page_content in the metadata
+        doc.metadata['original_text'] = doc.page_content
+
+        # Add title to page_content
+        if 'title' in doc.metadata:
+            title = doc.metadata['title']
+            doc.page_content = f'Titre: {title}\n\n{doc.page_content}'
+    return splitted_docs
+
+
 def em_settings_from_config(setting_dict: dict) -> BaseEMSetting:
     """Get embeddings settings from config dict."""
     # Create settings class according to embeddings provider from config file