From eb57e75450f716e213a2187f57f31db2f50af898 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Thu, 7 Nov 2024 11:32:23 -0800 Subject: [PATCH] fix: parallel --- llama-index-core/llama_index/core/readers/file/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llama-index-core/llama_index/core/readers/file/base.py b/llama-index-core/llama_index/core/readers/file/base.py index b015458ce9bad..3a9ff22f9fe95 100644 --- a/llama-index-core/llama_index/core/readers/file/base.py +++ b/llama-index-core/llama_index/core/readers/file/base.py @@ -711,12 +711,11 @@ def load_data( num_workers = num_cpus with multiprocessing.get_context("spawn").Pool(num_workers) as p: - results = [] + async_results = [] for input_file in files_to_process: - results.append( - p.apply( + async_results.append( + p.apply_async( SimpleDirectoryReader.load_file, - args=(), kwds={ "input_file": input_file, "file_metadata": self.file_metadata, @@ -729,6 +728,7 @@ def load_data( }, ) ) + results = [res.get() for res in async_results] documents = reduce(lambda x, y: x + y, results) else: