diff --git a/.gitignore b/.gitignore index 604515e..2901232 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,7 @@ data .mr.developer.cfg #PyCharm -.idea/ \ No newline at end of file +.idea/ + +#Sublime +*.sublime-* diff --git a/README.md b/README.md index 68a92ee..b499fe4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ The next steps are provide support to: * authentication * nearest preference in replica sets * gridfs -* all python versions (2.5, 2.6, 2.7, 3.2 and PyPy), only python 2.7 is tested now +* all python versions (2.6, 2.7, 3.2 and PyPy), only python 2.7 is tested now ## Documentation @@ -67,7 +67,7 @@ class Handler(tornado.web.RequestHandler): def get(self): user = {'_id': ObjectId(), 'name': 'User Name'} yield gen.Task(self.db.user.insert, user) - + yield gen.Task(self.db.user.update, user['_id'], {"$set": {'name': 'New User Name'}}) user_found = yield gen.Task(self.db.user.find_one, user['_id']) @@ -98,10 +98,10 @@ class Handler(tornado.web.RequestHandler): @gen.engine def get(self): user = {'_id': ObjectId()} - + # write on primary yield gen.Task(self.db.user.insert, user) - + # wait for replication time.sleep(2) @@ -191,4 +191,4 @@ make test ## Issues -Please report any issues via [github issues](https://github.com/marcelnicolay/mongotor/issues) \ No newline at end of file +Please report any issues via [github issues](https://github.com/marcelnicolay/mongotor/issues) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 0b2ea45..9f3f7a6 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -12,7 +12,7 @@ MongoTor supports installation using standard Python "distutils" or Install via easy_install or pip ------------------------------- -When ``easy_install`` or ``pip`` is available, the distribution can be +When ``easy_install`` or ``pip`` is available, the distribution can be downloaded from Pypi and installed in one step:: easy_install mongotor @@ -39,12 +39,14 @@ Python prompt like this: .. sourcecode:: python - >>> import mongotor + >>> import mongotor >>> mongotor.version # doctest: +SKIP Requirements ------------ +Python version 2.6+ is required. But only version 2.7 is tested for now. + The following three python libraries are required. * `pymongo `_ version 1.9+ for bson library @@ -52,4 +54,4 @@ The following three python libraries are required. .. note:: The above requirements are automatically managed when installed using - any of the supported installation methods \ No newline at end of file + any of the supported installation methods diff --git a/mongotor/__init__.py b/mongotor/__init__.py index bb8a479..ec9f7d0 100644 --- a/mongotor/__init__.py +++ b/mongotor/__init__.py @@ -15,4 +15,4 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -version = "0.1.0" +version = "0.1.5" diff --git a/mongotor/cursor.py b/mongotor/cursor.py index 5b43252..17eebb8 100644 --- a/mongotor/cursor.py +++ b/mongotor/cursor.py @@ -95,8 +95,8 @@ def find(self, callback=None): else: callback((response['data'], None)) - @gen.engine - def count(self, callback): + @gen.coroutine + def count(self): """Get the size of the results set for this query. Returns the number of documents in the results set for this query. Does @@ -110,10 +110,10 @@ def count(self, callback): if response and len(response) > 0 and 'n' in response: total = int(response['n']) - callback(total) + raise gen.Return(total) - @gen.engine - def distinct(self, key, callback): + @gen.coroutine + def distinct(self, key): """Get a list of distinct values for `key` among all documents in the result set of this query. @@ -131,7 +131,7 @@ def distinct(self, key, callback): response, error = yield gen.Task(self._database.command, 'distinct', self._collection, **command) - callback(response['values']) + raise gen.Return(response['values']) def _query_options(self): """Get the query options string to use for this query.""" diff --git a/mongotor/database.py b/mongotor/database.py index b28ea75..79f5579 100644 --- a/mongotor/database.py +++ b/mongotor/database.py @@ -29,7 +29,7 @@ def initialized(fn): @wraps(fn) def wrapped(self, *args, **kwargs): - if not hasattr(self, '_initialized'): + if not hasattr(self, '_initialized') or not self._initialized: raise DatabaseError("you must be initialize database before perform this action") return fn(self, *args, **kwargs) @@ -45,6 +45,7 @@ class Database(object): def __new__(cls): if not cls._instance: cls._instance = super(Database, cls).__new__(cls) + cls._initialized = False return cls._instance @@ -147,7 +148,8 @@ def disconnect(cls): >>> Database.disconnect() """ - if not cls._instance or not hasattr(cls._instance, '_initialized'): + if (not cls._instance or not hasattr(cls._instance, '_initialized') + or not cls._instance._initialized): raise ValueError("Database isn't initialized") for node in cls._instance._nodes: diff --git a/mongotor/message.py b/mongotor/message.py index 6cd3226..c857dbf 100644 --- a/mongotor/message.py +++ b/mongotor/message.py @@ -28,7 +28,7 @@ from mongotor.errors import InvalidOperationError -__ZERO = "\x00\x00\x00\x00" +__ZERO = b"\x00\x00\x00\x00" def __last_error(args): @@ -57,7 +57,7 @@ def insert(collection_name, docs, check_keys, safe, last_error_args): """ data = __ZERO data += bson._make_c_string(collection_name) - bson_data = "".join([bson.BSON.encode(doc, check_keys) for doc in docs]) + bson_data = b"".join([bson.BSON.encode(doc, check_keys) for doc in docs]) if not bson_data: raise InvalidOperationError("cannot do an empty bulk insert") data += bson_data diff --git a/mongotor/orm/collection.py b/mongotor/orm/collection.py index 985125a..9292782 100644 --- a/mongotor/orm/collection.py +++ b/mongotor/orm/collection.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +import re import logging from tornado import gen from mongotor.client import Client @@ -32,6 +33,9 @@ class CollectionMetaClass(type): def __new__(cls, name, bases, attrs): + if not attrs.get('__collection__'): + attrs['__collection__'] = re.sub( + r'\B([A-Z]+)', r'_\1', name).lower() global __lazy_classes__ # Add the document's fields to the _data @@ -65,6 +69,10 @@ class Collection(object): >>> class Users(collection.Collection): >>> __collection__ = 'users' >>> name = field.StringField() + + If you do not specify `__collection__` attribute, it is + auto-generated from class name. Camel case is converted + to snake case. For example: CamelCase -> camel_case. """ __metaclass__ = CollectionMetaClass @@ -133,8 +141,11 @@ def create(cls, dictionary, cleaned=False): return instance - @gen.engine - def save(self, safe=True, check_keys=True, callback=None): + def get_client(self): + return Client(Database(), self.__collection__) + + @gen.coroutine + def save(self, safe=True, check_keys=True): """Save a document >>> user = Users() @@ -150,7 +161,7 @@ def save(self, safe=True, check_keys=True, callback=None): """ pre_save.send(instance=self) - client = Client(Database(), self.__collection__) + client = self.get_client() response, error = yield gen.Task(client.insert, self.as_dict(), safe=safe, check_keys=check_keys) @@ -158,11 +169,10 @@ def save(self, safe=True, check_keys=True, callback=None): post_save.send(instance=self) - if callback: - callback((response, error)) + raise gen.Return((response, error)) - @gen.engine - def remove(self, safe=True, callback=None): + @gen.coroutine + def remove(self, safe=True): """Remove a document :Parameters: @@ -171,17 +181,15 @@ def remove(self, safe=True, callback=None): """ pre_remove.send(instance=self) - client = Client(Database(), self.__collection__) + client = self.get_client() response, error = yield gen.Task(client.remove, self._id, safe=safe) post_remove.send(instance=self) - if callback: - callback((response, error)) + raise gen.Return((response, error)) - @gen.engine - def update(self, document=None, upsert=False, safe=True, multi=False, - callback=None, force=False): + @gen.coroutine + def update(self, document=None, upsert=False, safe=True, multi=False, force=False): """Update a document :Parameters: @@ -190,8 +198,7 @@ def update(self, document=None, upsert=False, safe=True, multi=False, - `force`: if True will overide full document """ if not document and not self.dirty_fields: - callback(tuple()) - return + raise gen.Return(tuple()) pre_update.send(instance=self) @@ -201,7 +208,7 @@ def update(self, document=None, upsert=False, safe=True, multi=False, else: document = {"$set": self.as_dict(self.dirty_fields)} - client = Client(Database(), self.__collection__) + client = self.get_client() spec = {'_id': self._id} response, error = yield gen.Task(client.update, spec, document, @@ -211,5 +218,4 @@ def update(self, document=None, upsert=False, safe=True, multi=False, post_update.send(instance=self) - if callback: - callback((response, error)) + raise gen.Return((response, error)) diff --git a/mongotor/orm/field.py b/mongotor/orm/field.py index ff57dc4..75f3b48 100644 --- a/mongotor/orm/field.py +++ b/mongotor/orm/field.py @@ -67,7 +67,7 @@ def __init__(self, regex=None, *args, **kwargs): def _validate(self, value): value = super(StringField, self)._validate(value) - if self.regex is not None and self.regex.match(value) is None: + if value is not None and self.regex is not None and self.regex.match(value) is None: raise(TypeError("Value did not match regex")) return value @@ -116,11 +116,19 @@ def __init__(self, field_type, min_value=None, max_value=None, def _validate(self, value): value = super(NumberField, self)._validate(value) + if value is None: + if self.min_value is not None: + value = self.min_value + elif self.max_value is not None: + value = self.max_value + else: + value = self.field_type() + if self.min_value is not None and value < self.min_value: raise(TypeError("Value can not be less than %s" % (self.min_value))) if self.max_value is not None and value > self.max_value: - raise(TypeError("Value can not be more than %s" & (self.max_value))) + raise(TypeError("Value can not be more than %s" % (self.max_value))) return value diff --git a/mongotor/orm/manager.py b/mongotor/orm/manager.py index 7a76f9f..aee23b6 100644 --- a/mongotor/orm/manager.py +++ b/mongotor/orm/manager.py @@ -26,8 +26,8 @@ class Manager(object): def __init__(self, collection): self.collection = collection - @gen.engine - def find_one(self, query, callback): + @gen.coroutine + def find_one(self, query): client = Client(Database(), self.collection.__collection__) result, error = yield gen.Task(client.find_one, query) @@ -35,10 +35,10 @@ def find_one(self, query, callback): if result: instance = self.collection.create(result, cleaned=True) - callback(instance) + raise gen.Return(instance) - @gen.engine - def find(self, query, callback, **kw): + @gen.coroutine + def find(self, query, **kw): client = Client(Database(), self.collection.__collection__) result, error = yield gen.Task(client.find, query, **kw) @@ -48,20 +48,38 @@ def find(self, query, callback, **kw): for item in result: items.append(self.collection.create(item, cleaned=True)) - callback(items) + raise gen.Return(items) - def count(self, query=None, callback=None): + @gen.coroutine + def remove(self, *args, **kwargs): client = Client(Database(), self.collection.__collection__) - client.find(query).count(callback=callback) + result, error = yield gen.Task(client.remove, *args, **kwargs) + raise gen.Return(result) - @gen.engine - def distinct(self, key, callback, query=None): + @gen.coroutine + def all(self): + """Find all documents + + This method is alias for `find({})` + """ + result = yield self.find({}) + raise gen.Return(result) + + @gen.coroutine + def count(self, query=None): + client = Client(Database(), self.collection.__collection__) + count = yield client.find(query).count() + raise gen.Return(count) + + @gen.coroutine + def distinct(self, key, query=None): client = Client(Database(), self.collection.__collection__) - client.find(query).distinct(key, callback=callback) + result = yield client.find(query).distinct(key) + raise gen.Return(result) - @gen.engine + @gen.coroutine def geo_near(self, near, max_distance=None, num=None, spherical=None, - unique_docs=None, query=None, callback=None, **kw): + unique_docs=None, query=None, **kw): command = SON({"geoNear": self.collection.__collection__}) @@ -90,10 +108,10 @@ def geo_near(self, near, max_distance=None, num=None, spherical=None, for item in result['results']: items.append(self.collection.create(item['obj'], cleaned=True)) - callback(items) + raise gen.Return(items) - @gen.engine - def map_reduce(self, map_, reduce_, callback, query=None, out=None): + @gen.coroutine + def map_reduce(self, map_, reduce_, query=None, out=None): command = SON({'mapreduce': self.collection.__collection__}) command.update({ @@ -108,15 +126,11 @@ def map_reduce(self, map_, reduce_, callback, query=None, out=None): result, error = yield gen.Task(Database().command, command) if not result or int(result['ok']) != 1: - callback(None) - return - - callback(result['results']) + raise gen.Return(None) - @gen.engine - def truncate(self, callback=None): - client = Client(Database(), self.collection.__collection__) - yield gen.Task(client.remove, {}) + raise gen.Return(result['results']) - if callback: - callback() + @gen.coroutine + def truncate(self): + result = yield self.remove() + raise gen.Return(result) diff --git a/setup.py b/setup.py index a20f913..44d84ff 100644 --- a/setup.py +++ b/setup.py @@ -14,16 +14,16 @@ def get_packages(): return packages setup( - name = 'mongotor', - version = version, - description = "(MongoDB + Tornado) is an asynchronous driver and toolkit for working with MongoDB inside a Tornado app", - long_description = open("README.md").read(), - keywords = ['mongo','tornado'], - author = 'Marcel Nicolay', - author_email = 'marcel.nicolay@gmail.com', - url = 'http://marcelnicolay.github.com/mongotor/', - license = 'OSI', - classifiers = ['Development Status :: 4 - Beta', + name='mongotor', + version=version, + description="(MongoDB + Tornado) is an asynchronous driver and toolkit for working with MongoDB inside a Tornado app", + long_description=open("README.md").read(), + keywords=['mongo', 'tornado'], + author='Marcel Nicolay', + author_email='marcel.nicolay@gmail.com', + url='http://marcelnicolay.github.com/mongotor/', + license='OSI', + classifiers=['Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved', 'Natural Language :: English', @@ -32,7 +32,8 @@ def get_packages(): 'Programming Language :: Python :: 2.7', 'Topic :: Software Development :: Libraries :: Application Frameworks', ], - install_requires = open("requirements.txt").read().split("\n"), - packages = get_packages(), - test_suite="nose.collector" -) \ No newline at end of file + install_requires=open("requirements.txt").read().split("\n"), + packages=get_packages(), + test_suite="nose.collector", + use_2to3=True +) diff --git a/tests/orm/test_collection.py b/tests/orm/test_collection.py index 1d875a2..99c4022 100644 --- a/tests/orm/test_collection.py +++ b/tests/orm/test_collection.py @@ -177,8 +177,8 @@ class CollectionTest(Collection): __collection__ = 'collection_test' Database.disconnect() - CollectionTest().save.when.called_with(callback=None) \ - .throw(DatabaseError, 'you must be initialize database before perform this action') + CollectionTest().save(callback=None) + self.assertRaises(DatabaseError, self.wait) Database.init(["localhost:27027", "localhost:27028"], dbname='test') diff --git a/tests/orm/test_manager.py b/tests/orm/test_manager.py index 768d61f..b546542 100644 --- a/tests/orm/test_manager.py +++ b/tests/orm/test_manager.py @@ -88,6 +88,84 @@ def test_find_not_found(self): collections_found.should.have.length_of(0) + def test_remove_all(self): + """[ManagerTestCase] - Remove all documents from collection""" + collection_test = CollectionTest() + collection_test._id = ObjectId() + collection_test.string_attr = "string value" + collection_test.save(callback=self.stop) + self.wait() + + other_collection_test = CollectionTest() + other_collection_test._id = ObjectId() + other_collection_test.string_attr = "other string value" + other_collection_test.save(callback=self.stop) + self.wait() + + CollectionTest.objects.all(callback=self.stop) + collections_found = self.wait() + collections_found.should.have.length_of(2) + + CollectionTest.objects.remove(callback=self.stop) + result = self.wait() + + CollectionTest.objects.all(callback=self.stop) + collections_found = self.wait() + collections_found.should.have.length_of(0) + + def test_remove_one_by_id(self): + """[ManagerTestCase] - Remove one document from collection by id""" + collection_test = CollectionTest() + collection_test._id = ObjectId() + collection_test.string_attr = "string value" + collection_test.save(callback=self.stop) + self.wait() + + other_collection_test = CollectionTest() + other_collection_test._id = ObjectId() + other_collection_test.string_attr = "other string value" + other_collection_test.save(callback=self.stop) + self.wait() + + CollectionTest.objects.all(callback=self.stop) + collections_found = self.wait() + collections_found.should.have.length_of(2) + + CollectionTest.objects.remove(collection_test._id, callback=self.stop) + result = self.wait() + + CollectionTest.objects.all(callback=self.stop) + collections_found = self.wait() + collections_found.should.have.length_of(1) + collections_found[0]._id.should.be.equal(other_collection_test._id) + + def test_remove_one_by_spec(self): + """[ManagerTestCase] - Remove one document from collection by spec""" + collection_test = CollectionTest() + collection_test._id = ObjectId() + collection_test.string_attr = "string value" + collection_test.save(callback=self.stop) + self.wait() + + other_collection_test = CollectionTest() + other_collection_test._id = ObjectId() + other_collection_test.string_attr = "other string value" + other_collection_test.save(callback=self.stop) + self.wait() + + CollectionTest.objects.all(callback=self.stop) + collections_found = self.wait() + collections_found.should.have.length_of(2) + + CollectionTest.objects.remove({"string_attr": "other string value"}, + callback=self.stop) + result = self.wait() + + CollectionTest.objects.all(callback=self.stop) + collections_found = self.wait() + collections_found.should.have.length_of(1) + collections_found[0]._id.should.be.equal(collection_test._id) + def test_count(self): """[ManagerTestCase] - Count document in collection""" collection_test = CollectionTest() @@ -223,4 +301,4 @@ def test_execute_simple_mapreduce_return_results_inline(self): self.assertEquals({u'_id': u'Value A', u'value': 2.0}, results[0]) self.assertEquals({u'_id': u'Value B', u'value': 1.0}, results[1]) self.assertEquals({u'_id': u'Value C', u'value': 1.0}, results[2]) - self.assertEquals({u'_id': u'Value D', u'value': 1.0}, results[3]) \ No newline at end of file + self.assertEquals({u'_id': u'Value D', u'value': 1.0}, results[3])