fix: parallel

run-llama · Nov 7, 2024 · eb57e75 · eb57e75
1 parent 7735e5c
commit eb57e75
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/llama-index-core/llama_index/core/readers/file/base.py b/llama-index-core/llama_index/core/readers/file/base.py
@@ -711,12 +711,11 @@ def load_data(
                 num_workers = num_cpus
 
             with multiprocessing.get_context("spawn").Pool(num_workers) as p:
-                results = []
+                async_results = []
                 for input_file in files_to_process:
-                    results.append(
-                        p.apply(
+                    async_results.append(
+                        p.apply_async(
                             SimpleDirectoryReader.load_file,
-                            args=(),
                             kwds={
                                 "input_file": input_file,
                                 "file_metadata": self.file_metadata,
@@ -729,6 +728,7 @@ def load_data(
                             },
                         )
                     )
+                results = [res.get() for res in async_results]
                 documents = reduce(lambda x, y: x + y, results)
 
         else: