From 7cc21a0ca02da3cb25318f593b88283c09d54c62 Mon Sep 17 00:00:00 2001 From: NoahCarnahan Date: Thu, 1 Dec 2016 12:56:35 +0530 Subject: [PATCH 1/6] Add safe generator class and decorator --- dimagi/utils/couch/database.py | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/dimagi/utils/couch/database.py b/dimagi/utils/couch/database.py index b683e71..22735c9 100644 --- a/dimagi/utils/couch/database.py +++ b/dimagi/utils/couch/database.py @@ -1,3 +1,5 @@ +from functools import wraps + from couchdbkit import ResourceConflict from couchdbkit.client import Database from dimagi.ext.couchdbkit import Document @@ -8,6 +10,51 @@ from time import sleep +class GeneratorAlreadyConsumedException(Exception): + pass + + +class SafeGenerator(object): + """ + This generator will raise a GeneratorAlreadyConsumedException if a user attempts to iterate through this + generator a second time. + """ + def __init__(self, generator): + """ + Create a new SafeGenerator by wrapping a normal one + :param generator: the generator to be wrapped + """ + self._generator = generator + self._has_been_consumed = False + + def __iter__(self): + return self + + def next(self): + if self._has_been_consumed: + raise GeneratorAlreadyConsumedException + try: + return self._generator.next() + except StopIteration as e: + self._has_been_consumed = True + raise + + def __next__(self): + # Python 3 compatibility + return self.next() + + +def safe_generator(fn): + """ + This decorator function wraps the return value of the given function in a SafeGenerator. + You should only use this decorator on functions that return generators. + """ + @wraps(fn) + def wrapper(*args, **kwargs): + return SafeGenerator(fn(*args, **kwargs)) + return wrapper + + class DocTypeMismatchException(Exception): pass @@ -58,12 +105,15 @@ def get_view_names(database): views.append("%s/%s" % (doc.name, view_name)) return views + +@safe_generator def iter_docs(database, ids, chunksize=100, **query_params): for doc_ids in chunked(ids, chunksize): for doc in get_docs(database, keys=doc_ids, **query_params): yield doc +@safe_generator def iter_docs_with_retry(database, ids, chunksize=100, max_attempts=5, **query_params): """ A version of iter_docs that retries fetching documents if the connection From fb5c0849491498ce3f09e4c1a063aeee5bbd6025 Mon Sep 17 00:00:00 2001 From: NoahCarnahan Date: Thu, 1 Dec 2016 13:05:35 +0530 Subject: [PATCH 2/6] Add test --- dimagi/utils/tests/database.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 dimagi/utils/tests/database.py diff --git a/dimagi/utils/tests/database.py b/dimagi/utils/tests/database.py new file mode 100644 index 0000000..46bfbec --- /dev/null +++ b/dimagi/utils/tests/database.py @@ -0,0 +1,14 @@ +from django.test import SimpleTestCase + +from dimagi.utils.couch.database import SafeGenerator, GeneratorAlreadyConsumedException + + +class SafeGeneratorTest(SimpleTestCase): + + def test_exception(self): + generator = (x for x in range(3)) + safe_generator = SafeGenerator(generator) + self.assertEqual(list(safe_generator), [0,1,2]) + with self.assertRaises(GeneratorAlreadyConsumedException): + for i in safe_generator: + pass From 35f2abae17a1636236d0e7ea986a20fa85d3d85c Mon Sep 17 00:00:00 2001 From: NoahCarnahan Date: Mon, 5 Dec 2016 10:56:30 -0500 Subject: [PATCH 3/6] Rename classes --- dimagi/utils/couch/database.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/dimagi/utils/couch/database.py b/dimagi/utils/couch/database.py index 22735c9..e12a825 100644 --- a/dimagi/utils/couch/database.py +++ b/dimagi/utils/couch/database.py @@ -10,18 +10,26 @@ from time import sleep -class GeneratorAlreadyConsumedException(Exception): +class IteratorAlreadyConsumedException(Exception): pass -class SafeGenerator(object): +class SafeIterator(object): """ - This generator will raise a GeneratorAlreadyConsumedException if a user attempts to iterate through this - generator a second time. + This iterator-like object wraps an iterator. + The SafeIterator will raise a IteratorAlreadyConsumedException if a user attempts to iterate through it a + second time. + This is useful if the wrapped iterator is a one time user generator, which would simply raise StopIteration + exception again (thereby, appearing "empty" to a user who didn't realize that the generator had already been + consumed). + + Note that this class cannot be used interchangeably with an iterator, because iterators must raise + StopIteration exceptions on subsequent iterations. However, in practice, I don't expect this to be an issue. + https://docs.python.org/2/library/stdtypes.html#iterator-types """ def __init__(self, generator): """ - Create a new SafeGenerator by wrapping a normal one + Create a new SafeIterator by wrapping a normal generator (or iterator) :param generator: the generator to be wrapped """ self._generator = generator @@ -32,7 +40,7 @@ def __iter__(self): def next(self): if self._has_been_consumed: - raise GeneratorAlreadyConsumedException + raise IteratorAlreadyConsumedException try: return self._generator.next() except StopIteration as e: @@ -46,12 +54,14 @@ def __next__(self): def safe_generator(fn): """ - This decorator function wraps the return value of the given function in a SafeGenerator. + This decorator function wraps the return value of the given function in a SafeIterator. You should only use this decorator on functions that return generators. + The SafeIterator will render the generator "safe" by raising a IteratorAlreadyConsumedException if it is + iterated a second time. """ @wraps(fn) def wrapper(*args, **kwargs): - return SafeGenerator(fn(*args, **kwargs)) + return SafeIterator(fn(*args, **kwargs)) return wrapper From 0fad0db3f9d26655afbd3543b1f9dfb658f5cde2 Mon Sep 17 00:00:00 2001 From: NoahCarnahan Date: Mon, 5 Dec 2016 11:06:36 -0500 Subject: [PATCH 4/6] Move to a new file --- dimagi/utils/couch/database.py | 57 +--------------------------------- dimagi/utils/safe_iterator.py | 56 +++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 56 deletions(-) create mode 100644 dimagi/utils/safe_iterator.py diff --git a/dimagi/utils/couch/database.py b/dimagi/utils/couch/database.py index e12a825..958bd35 100644 --- a/dimagi/utils/couch/database.py +++ b/dimagi/utils/couch/database.py @@ -1,5 +1,3 @@ -from functools import wraps - from couchdbkit import ResourceConflict from couchdbkit.client import Database from dimagi.ext.couchdbkit import Document @@ -9,60 +7,7 @@ from requests.exceptions import RequestException from time import sleep - -class IteratorAlreadyConsumedException(Exception): - pass - - -class SafeIterator(object): - """ - This iterator-like object wraps an iterator. - The SafeIterator will raise a IteratorAlreadyConsumedException if a user attempts to iterate through it a - second time. - This is useful if the wrapped iterator is a one time user generator, which would simply raise StopIteration - exception again (thereby, appearing "empty" to a user who didn't realize that the generator had already been - consumed). - - Note that this class cannot be used interchangeably with an iterator, because iterators must raise - StopIteration exceptions on subsequent iterations. However, in practice, I don't expect this to be an issue. - https://docs.python.org/2/library/stdtypes.html#iterator-types - """ - def __init__(self, generator): - """ - Create a new SafeIterator by wrapping a normal generator (or iterator) - :param generator: the generator to be wrapped - """ - self._generator = generator - self._has_been_consumed = False - - def __iter__(self): - return self - - def next(self): - if self._has_been_consumed: - raise IteratorAlreadyConsumedException - try: - return self._generator.next() - except StopIteration as e: - self._has_been_consumed = True - raise - - def __next__(self): - # Python 3 compatibility - return self.next() - - -def safe_generator(fn): - """ - This decorator function wraps the return value of the given function in a SafeIterator. - You should only use this decorator on functions that return generators. - The SafeIterator will render the generator "safe" by raising a IteratorAlreadyConsumedException if it is - iterated a second time. - """ - @wraps(fn) - def wrapper(*args, **kwargs): - return SafeIterator(fn(*args, **kwargs)) - return wrapper +from dimagi.utils.safe_iterator import safe_generator class DocTypeMismatchException(Exception): diff --git a/dimagi/utils/safe_iterator.py b/dimagi/utils/safe_iterator.py new file mode 100644 index 0000000..4235727 --- /dev/null +++ b/dimagi/utils/safe_iterator.py @@ -0,0 +1,56 @@ +from functools import wraps + + +class IteratorAlreadyConsumedException(Exception): + pass + + +class SafeIterator(object): + """ + This iterator-like object wraps an iterator. + The SafeIterator will raise a IteratorAlreadyConsumedException if a user attempts to iterate through it a + second time. + This is useful if the wrapped iterator is a one time user generator, which would simply raise StopIteration + exception again (thereby, appearing "empty" to a user who didn't realize that the generator had already been + consumed). + + Note that this class cannot be used interchangeably with an iterator, because iterators must raise + StopIteration exceptions on subsequent iterations. However, in practice, I don't expect this to be an issue. + https://docs.python.org/2/library/stdtypes.html#iterator-types + """ + def __init__(self, generator): + """ + Create a new SafeIterator by wrapping a normal generator (or iterator) + :param generator: the generator to be wrapped + """ + self._generator = generator + self._has_been_consumed = False + + def __iter__(self): + return self + + def next(self): + if self._has_been_consumed: + raise IteratorAlreadyConsumedException + try: + return self._generator.next() + except StopIteration as e: + self._has_been_consumed = True + raise + + def __next__(self): + # Python 3 compatibility + return self.next() + + +def safe_generator(fn): + """ + This decorator function wraps the return value of the given function in a SafeIterator. + You should only use this decorator on functions that return generators. + The SafeIterator will render the generator "safe" by raising a IteratorAlreadyConsumedException if it is + iterated a second time. + """ + @wraps(fn) + def wrapper(*args, **kwargs): + return SafeIterator(fn(*args, **kwargs)) + return wrapper From 69b2c81d77eaad0112adc558d1716d046d5c9f4e Mon Sep 17 00:00:00 2001 From: NoahCarnahan Date: Mon, 5 Dec 2016 11:11:05 -0500 Subject: [PATCH 5/6] rename test file --- dimagi/utils/tests/{database.py => sage_iterator.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dimagi/utils/tests/{database.py => sage_iterator.py} (100%) diff --git a/dimagi/utils/tests/database.py b/dimagi/utils/tests/sage_iterator.py similarity index 100% rename from dimagi/utils/tests/database.py rename to dimagi/utils/tests/sage_iterator.py From 4666dbc692ffd67da9f81da69856a4c2014fa1f8 Mon Sep 17 00:00:00 2001 From: NoahCarnahan Date: Mon, 5 Dec 2016 11:11:31 -0500 Subject: [PATCH 6/6] Update test and add decorator test --- dimagi/utils/tests/sage_iterator.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/dimagi/utils/tests/sage_iterator.py b/dimagi/utils/tests/sage_iterator.py index 46bfbec..19f3931 100644 --- a/dimagi/utils/tests/sage_iterator.py +++ b/dimagi/utils/tests/sage_iterator.py @@ -1,14 +1,24 @@ from django.test import SimpleTestCase -from dimagi.utils.couch.database import SafeGenerator, GeneratorAlreadyConsumedException +from dimagi.utils.safe_iterator import SafeIterator, IteratorAlreadyConsumedException, safe_generator -class SafeGeneratorTest(SimpleTestCase): +class SafeIteratorTest(SimpleTestCase): def test_exception(self): generator = (x for x in range(3)) - safe_generator = SafeGenerator(generator) - self.assertEqual(list(safe_generator), [0,1,2]) - with self.assertRaises(GeneratorAlreadyConsumedException): + safe_generator = SafeIterator(generator) + self.assertEqual(list(safe_generator), [0, 1, 2]) + with self.assertRaises(IteratorAlreadyConsumedException): for i in safe_generator: pass + + def test_decorator(self): + @safe_generator + def func(): + return (x for x in range(3)) + + generator = func() + self.assertEqual(list(generator), [0, 1, 2]) + with self.assertRaises(IteratorAlreadyConsumedException): + list(generator)