Skip to content

Commit

Permalink
change exhausted
Browse files Browse the repository at this point in the history
  • Loading branch information
avishaihalev committed Jun 17, 2024
1 parent cca0d45 commit 6205cac
Showing 1 changed file with 9 additions and 4 deletions.
13 changes: 9 additions & 4 deletions mmap_ninja/src/mmap_ninja/parallel.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from enum import Enum
from functools import partial
from tqdm.auto import tqdm

Expand All @@ -10,7 +11,11 @@
HAS_JOBLIB = True


EXHAUSTED = '__EXHAUSTED__'
class _Exhausted(Enum):
exhausted = 'EXHAUSTED'


EXHAUSTED = _Exhausted.exhausted


class ParallelBatchCollector:
Expand Down Expand Up @@ -84,7 +89,7 @@ def _collect_no_parallel_batch(self):
results = [_get_from_indexable(self.indexable, j) for j in self._rng()]

if self.exhausted(results):
results = [r for r in results if not isinstance(r, str) or r != EXHAUSTED]
results = [r for r in results if r is not EXHAUSTED]

return results

Expand All @@ -94,15 +99,15 @@ def _collect_parallel_batch(self):
results = self._parallel(func(j) for j in self._rng())

if self.exhausted(results):
results = [r for r in results if not isinstance(r, str) or r != EXHAUSTED]
results = [r for r in results if r is not EXHAUSTED]
self._parallel.__exit__(None, None, None)

return results

def exhausted(self, results=()):
self._exhausted = (
self._exhausted or
any(isinstance(r, str) and r == EXHAUSTED for r in results) or
any(r is EXHAUSTED for r in results) or
self.completed_batches()
)

Expand Down

0 comments on commit 6205cac

Please sign in to comment.