From 1849f75ad04a8a50c73862a6d0794fe65016343c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 31 Aug 2010 00:23:59 +0100 Subject: [PATCH 1/8] Made GridFSProxy a bit stricter / safer --- mongoengine/fields.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ffcfb53d..418f57cc 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -511,6 +511,10 @@ class BinaryField(BaseField): raise ValidationError('Binary value is too long') +class GridFSError(Exception): + pass + + class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS """ @@ -541,12 +545,18 @@ class GridFSProxy(object): self.grid_id = self.newfile._id def put(self, file, **kwargs): + if self.grid_id: + raise GridFSError('This document alreay has a file. Either delete ' + 'it or call replace to overwrite it') self.grid_id = self.fs.put(file, **kwargs) def write(self, string): - if not self.newfile: + if self.grid_id: + if not self.newfile: + raise GridFSError('This document alreay has a file. Either ' + 'delete it or call replace to overwrite it') + else: self.new_file() - self.grid_id = self.newfile._id self.newfile.write(string) def writelines(self, lines): From 2af5f3c56ebc7feff6a0a1cd95a77e12b425efed Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 31 Aug 2010 00:24:30 +0100 Subject: [PATCH 2/8] Added support for querying by array position. Closes #36. --- mongoengine/queryset.py | 6 +++++- tests/queryset.py | 43 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 662fa8c3..b8ca125d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -344,6 +344,8 @@ class QuerySet(object): mongo_query = {} for key, value in query.items(): parts = key.split('__') + indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] + parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None if parts[-1] in operators + match_operators + geo_operators: @@ -381,7 +383,9 @@ class QuerySet(object): "been implemented" % op) elif op not in match_operators: value = {'$' + op: value} - + + for i, part in indices: + parts.insert(i, part) key = '.'.join(parts) if op is None or key not in mongo_query: mongo_query[key] = value diff --git a/tests/queryset.py b/tests/queryset.py index e3912246..3d714be2 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -165,8 +165,49 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.get(age__lt=30) self.assertEqual(person.name, "User A") + def test_find_array_position(self): + """Ensure that query by array position works. + """ + class Comment(EmbeddedDocument): + name = StringField() + + class Post(EmbeddedDocument): + comments = ListField(EmbeddedDocumentField(Comment)) + + class Blog(Document): + tags = ListField(StringField()) + posts = ListField(EmbeddedDocumentField(Post)) + + Blog.drop_collection() - + Blog.objects.create(tags=['a', 'b']) + self.assertEqual(len(Blog.objects(tags__0='a')), 1) + self.assertEqual(len(Blog.objects(tags__0='b')), 0) + self.assertEqual(len(Blog.objects(tags__1='a')), 0) + self.assertEqual(len(Blog.objects(tags__1='b')), 1) + + Blog.drop_collection() + + comment1 = Comment(name='testa') + comment2 = Comment(name='testb') + post1 = Post(comments=[comment1, comment2]) + post2 = Post(comments=[comment2, comment2]) + blog1 = Blog.objects.create(posts=[post1, post2]) + blog2 = Blog.objects.create(posts=[post2, post1]) + + blog = Blog.objects(posts__0__comments__0__name='testa').get() + self.assertEqual(blog, blog1) + + query = Blog.objects(posts__1__comments__1__name='testb') + self.assertEqual(len(query), 2) + + query = Blog.objects(posts__1__comments__1__name='testa') + self.assertEqual(len(query), 0) + + query = Blog.objects(posts__0__comments__1__name='testa') + self.assertEqual(len(query), 0) + + Blog.drop_collection() def test_get_or_create(self): """Ensure that ``get_or_create`` returns one result or creates a new From 449f5a00dccce95b3efebd59438ee981ddfb7435 Mon Sep 17 00:00:00 2001 From: Nicolas Perriault Date: Sat, 11 Sep 2010 17:45:57 +0200 Subject: [PATCH 3/8] added a 'validate' option to Document.save() +docs +tests --- docs/guide/defining-documents.rst | 14 ++++++++++++++ mongoengine/document.py | 6 ++++-- tests/document.py | 10 ++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c276869..dff3ed60 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -214,6 +214,20 @@ either a single field name, or a list or tuple of field names:: first_name = StringField() last_name = StringField(unique_with='first_name') +Skipping Document validation on save +------------------------------------ +You can also skip the whole document validation process by setting +``validate=False`` when caling the :meth:`~mongoengine.document.Document.save` +method:: + + class Recipient(Document): + name = StringField() + email = EmailField() + + recipient = Recipient(name='admin', email='root@localhost') + recipient.save() # will raise a ValidationError while + recipient.save(validate=False) # won't + Document collections ==================== Document classes that inherit **directly** from :class:`~mongoengine.Document` diff --git a/mongoengine/document.py b/mongoengine/document.py index e5dec145..af2a5e21 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -56,7 +56,7 @@ class Document(BaseDocument): __metaclass__ = TopLevelDocumentMetaclass - def save(self, safe=True, force_insert=False): + def save(self, safe=True, force_insert=False, validate=True): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. @@ -67,8 +67,10 @@ class Document(BaseDocument): :param safe: check if the operation succeeded before returning :param force_insert: only try to create a new document, don't allow updates of existing documents + :param validate: validates the document; set to ``False`` for skiping """ - self.validate() + if validate: + self.validate() doc = self.to_mongo() try: collection = self.__class__.objects._collection diff --git a/tests/document.py b/tests/document.py index 8bc907c5..80cf3f08 100644 --- a/tests/document.py +++ b/tests/document.py @@ -446,6 +446,16 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person_obj['name'], 'Test User') self.assertEqual(person_obj['age'], 30) self.assertEqual(person_obj['_id'], person.id) + # Test skipping validation on save + class Recipient(Document): + email = EmailField(required=True) + + recipient = Recipient(email='root@localhost') + self.assertRaises(ValidationError, recipient.save) + try: + recipient.save(validate=False) + except ValidationError: + fail() def test_delete(self): """Ensure that document may be deleted using the delete method. From f11ee1f9cf60979fbb5c6768219225202d510951 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 15 Sep 2010 09:47:13 +0100 Subject: [PATCH 4/8] Added support for using custom QuerySet classes --- mongoengine/base.py | 1 + mongoengine/queryset.py | 3 ++- tests/queryset.py | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 836817da..0cbd707d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -255,6 +255,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'index_background': False, 'index_drop_dups': False, 'index_opts': {}, + 'queryset_class': QuerySet, } meta.update(base_meta) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b8ca125d..8b486093 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -992,7 +992,8 @@ class QuerySetManager(object): self._collection = db[collection] # owner is the document that contains the QuerySetManager - queryset = QuerySet(owner, self._collection) + queryset_class = owner._meta['queryset_class'] or QuerySet + queryset = queryset_class(owner, self._collection) if self._manager_func: if self._manager_func.func_code.co_argcount == 1: queryset = self._manager_func(queryset) diff --git a/tests/queryset.py b/tests/queryset.py index 3d714be2..4491be8c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1378,6 +1378,26 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + def test_custom_querysets(self): + """Ensure that custom QuerySet classes may be used. + """ + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class Post(Document): + meta = {'queryset_class': CustomQuerySet} + + Post.drop_collection() + + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + if __name__ == '__main__': unittest.main() From 20dd7562e0e7307f3a53c77d33521cb2f826cf5c Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Thu, 16 Sep 2010 17:19:58 -0400 Subject: [PATCH 5/8] Adding multiprocessing support to mongoengine by using the identity of the process to define the connection. Each 'thread' gets its own pymongo connection. --- mongoengine/connection.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index ec3bf784..94cc6ea1 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,5 +1,5 @@ from pymongo import Connection - +import multiprocessing __all__ = ['ConnectionError', 'connect'] @@ -8,12 +8,12 @@ _connection_settings = { 'host': 'localhost', 'port': 27017, } -_connection = None +_connection = {} _db_name = None _db_username = None _db_password = None -_db = None +_db = {} class ConnectionError(Exception): @@ -22,32 +22,39 @@ class ConnectionError(Exception): def _get_connection(): global _connection + identity = get_identity() # Connect to the database if not already connected - if _connection is None: + if _connection.get(identity) is None: try: - _connection = Connection(**_connection_settings) + _connection[identity] = Connection(**_connection_settings) except: raise ConnectionError('Cannot connect to the database') - return _connection + return _connection[identity] def _get_db(): global _db, _connection + identity = get_identity() # Connect if not already connected - if _connection is None: - _connection = _get_connection() + if _connection.get(identity) is None: + _connection[identity] = _get_connection() - if _db is None: + if _db.get(identity) is None: # _db_name will be None if the user hasn't called connect() if _db_name is None: raise ConnectionError('Not connected to the database') # Get DB from current connection and authenticate if necessary - _db = _connection[_db_name] + _db[identity] = _connection[identity][_db_name] if _db_username and _db_password: - _db.authenticate(_db_username, _db_password) + _db[identity].authenticate(_db_username, _db_password) - return _db + return _db[identity] +def get_identity(): + identity = multiprocessing.current_process()._identity + identity = 0 if not identity else identity[0] + return identity + def connect(db, username=None, password=None, **kwargs): """Connect to the database specified by the 'db' argument. Connection settings may be provided here as well if the database is not running on From 73092dcb33ed64e64aedc3d32fa84780c6e534bb Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 19 Sep 2010 17:17:37 +0100 Subject: [PATCH 6/8] QuerySet update method returns num affected docs --- mongoengine/queryset.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b8ca125d..c199d64f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -766,7 +766,8 @@ class QuerySet(object): return mongo_update def update(self, safe_update=True, upsert=False, **update): - """Perform an atomic update on the fields matched by the query. + """Perform an atomic update on the fields matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. :param safe: check if the operation succeeded before returning :param update: Django-style update keyword arguments @@ -778,8 +779,10 @@ class QuerySet(object): update = QuerySet._transform_update(self._document, **update) try: - self._collection.update(self._query, update, safe=safe_update, - upsert=upsert, multi=True) + ret = self._collection.update(self._query, update, multi=True, + upsert=upsert, safe=safe_update) + if ret is not None and 'n' in ret: + return ret['n'] except pymongo.errors.OperationFailure, err: if unicode(err) == u'multi not coded yet': message = u'update() method requires MongoDB 1.1.3+' @@ -787,7 +790,8 @@ class QuerySet(object): raise OperationError(u'Update failed (%s)' % unicode(err)) def update_one(self, safe_update=True, upsert=False, **update): - """Perform an atomic update on first field matched by the query. + """Perform an atomic update on first field matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. :param safe: check if the operation succeeded before returning :param update: Django-style update keyword arguments @@ -799,11 +803,14 @@ class QuerySet(object): # Explicitly provide 'multi=False' to newer versions of PyMongo # as the default may change to 'True' if pymongo.version >= '1.1.1': - self._collection.update(self._query, update, safe=safe_update, - upsert=upsert, multi=False) + ret = self._collection.update(self._query, update, multi=False, + upsert=upsert, safe=safe_update) else: # Older versions of PyMongo don't support 'multi' - self._collection.update(self._query, update, safe=safe_update) + ret = self._collection.update(self._query, update, + safe=safe_update) + if ret is not None and 'n' in ret: + return ret['n'] except pymongo.errors.OperationFailure, e: raise OperationError(u'Update failed [%s]' % unicode(e)) From bb2487914987972c4d7ff6960de1012d53a7f566 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 19 Sep 2010 20:00:53 +0100 Subject: [PATCH 7/8] Moved custom queryset test to correct place --- tests/queryset.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 4491be8c..59a8216e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1307,6 +1307,26 @@ class QuerySetTest(unittest.TestCase): Event.drop_collection() + def test_custom_querysets(self): + """Ensure that custom QuerySet classes may be used. + """ + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class Post(Document): + meta = {'queryset_class': CustomQuerySet} + + Post.drop_collection() + + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + class QTest(unittest.TestCase): @@ -1378,26 +1398,6 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) - def test_custom_querysets(self): - """Ensure that custom QuerySet classes may be used. - """ - class CustomQuerySet(QuerySet): - def not_empty(self): - return len(self) > 0 - - class Post(Document): - meta = {'queryset_class': CustomQuerySet} - - Post.drop_collection() - - self.assertTrue(isinstance(Post.objects, CustomQuerySet)) - self.assertFalse(Post.objects.not_empty()) - - Post().save() - self.assertTrue(Post.objects.not_empty()) - - Post.drop_collection() - if __name__ == '__main__': unittest.main() From 98bc0a7c10f23c4c7582c222ec0d1432ebe5b567 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 25 Sep 2010 22:47:09 +0100 Subject: [PATCH 8/8] Raise AttributeError when necessary on GridFSProxy --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 418f57cc..87d52fd6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -528,6 +528,7 @@ class GridFSProxy(object): obj = self.get() if name in dir(obj): return getattr(obj, name) + raise AttributeError def __get__(self, instance, value): return self