fix: non-deterministic CI test failure

Makefile

@@ -28,3 +28,10 @@ clean:
 	rm -rf ./src/*.egg-info
 	rm -rf ./build
 	rm -rf ./dist
+
+install:
+	pip install -r requirements.txt
+	pip install -r requirements/test.txt
+	pip install -r requirements/docs.txt
+	pip install -r requirements/extras.txt
+	pip install -e .

src/litdata/processing/data_processor.py

@@ -1135,10 +1135,15 @@ def run(self, data_recipe: DataRecipe) -> None:
                     json.dump({"progress": str(100 * current_total * num_nodes / total_num_items) + "%"}, f)
 
             # Exit early if all the workers are done.
-            # This means there were some kinda of errors.
-            # TODO: Check whether this is still required.
+            # This means either there were some kinda of errors, or optimize function was very small.
             if all(not w.is_alive() for w in self.workers):
-                raise RuntimeError("One of the worker has failed")
+                for w in self.workers:
+                    if w.exitcode != 0:
+                        try:
+                            error = self.error_queue.get(timeout=0.01)
+                            self._exit_on_error(error)
+                        except Empty:
+                            continue
 
         if _TQDM_AVAILABLE:
             pbar.close()