diff --git a/src/litdata/processing/data_processor.py b/src/litdata/processing/data_processor.py index c5124c99..fae806b5 100644 --- a/src/litdata/processing/data_processor.py +++ b/src/litdata/processing/data_processor.py @@ -1140,7 +1140,11 @@ def run(self, data_recipe: DataRecipe) -> None: # Exit early if all the workers are done. # This means there were some kinda of errors. if all(not w.is_alive() for w in self.workers): - raise RuntimeError("One of the worker has failed") + try: + error = self.error_queue.get(timeout=0.001) + self._exit_on_error(error) + except Empty: + break if _TQDM_AVAILABLE: pbar.close()