From 2e718e113008de68f6b22c65e0edff2da6d6d955 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 11 Jun 2013 12:00:59 -0700 Subject: [PATCH 01/89] unit test showing the problem --- tests/document/instance.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/document/instance.py b/tests/document/instance.py index 81734aa0..8be2f618 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2351,6 +2351,24 @@ class InstanceTest(unittest.TestCase): system = System.objects.first() self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value) + def test_list_of_lists_of_references(self): + + class User(Document): + name = StringField() + + class Post(Document): + user_lists = ListField(ListField(ReferenceField(User))) + + User.drop_collection() + Post.drop_collection() + + u1 = User.objects.create(name='u1') + u2 = User.objects.create(name='u2') + u3 = User.objects.create(name='u3') + + Post.objects.create(user_lists=[[u1, u2], [u3]]) + self.assertEqual(Post.objects.all()[0].user_lists, [[u1, u2], [u3]]) + if __name__ == '__main__': unittest.main() From e0dd33e6be920e44c33d0fb8a1ceb32a3514cd72 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 11 Jun 2013 12:18:03 -0700 Subject: [PATCH 02/89] move the test into a more appropriate location --- tests/document/instance.py | 18 ------------------ tests/test_dereference.py | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 8be2f618..81734aa0 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2351,24 +2351,6 @@ class InstanceTest(unittest.TestCase): system = System.objects.first() self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value) - def test_list_of_lists_of_references(self): - - class User(Document): - name = StringField() - - class Post(Document): - user_lists = ListField(ListField(ReferenceField(User))) - - User.drop_collection() - Post.drop_collection() - - u1 = User.objects.create(name='u1') - u2 = User.objects.create(name='u2') - u3 = User.objects.create(name='u3') - - Post.objects.create(user_lists=[[u1, u2], [u3]]) - self.assertEqual(Post.objects.all()[0].user_lists, [[u1, u2], [u3]]) - if __name__ == '__main__': unittest.main() diff --git a/tests/test_dereference.py b/tests/test_dereference.py index e146963f..a63c7840 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -291,6 +291,30 @@ class FieldTest(unittest.TestCase): self.assertEqual(employee.friends, friends) self.assertEqual(q, 2) + def test_list_of_lists_of_references(self): + + class User(Document): + name = StringField() + + class Post(Document): + user_lists = ListField(ListField(ReferenceField(User))) + + class SimpleList(Document): + users = ListField(ReferenceField(User)) + + User.drop_collection() + Post.drop_collection() + + u1 = User.objects.create(name='u1') + u2 = User.objects.create(name='u2') + u3 = User.objects.create(name='u3') + + SimpleList.objects.create(users=[u1, u2, u3]) + self.assertEqual(SimpleList.objects.all()[0].users, [u1, u2, u3]) + + Post.objects.create(user_lists=[[u1, u2], [u3]]) + self.assertEqual(Post.objects.all()[0].user_lists, [[u1, u2], [u3]]) + def test_circular_reference(self): """Ensure you can handle circular references """ From a7ca9950fc3f4dc5b70124687bb3a5e5e57428c5 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 11 Jun 2013 15:36:35 -0700 Subject: [PATCH 03/89] potential fix for dereferencing nested lists --- mongoengine/dereference.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index e5e8886b..d822e430 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -35,7 +35,7 @@ class DeReference(object): if instance and isinstance(instance, (Document, TopLevelDocumentMetaclass)): doc_type = instance._fields.get(name) - if hasattr(doc_type, 'field'): + while hasattr(doc_type, 'field'): doc_type = doc_type.field if isinstance(doc_type, ReferenceField): @@ -50,9 +50,19 @@ class DeReference(object): return items elif not field.dbref: if not hasattr(items, 'items'): - items = [field.to_python(v) - if not isinstance(v, (DBRef, Document)) else v - for v in items] + + def _get_items(items): + new_items = [] + for v in items: + if isinstance(v, list): + new_items.append(_get_items(v)) + elif not isinstance(v, (DBRef, Document)): + new_items.append(field.to_python(v)) + else: + new_items.append(v) + return new_items + + items = _get_items(items) else: items = dict([ (k, field.to_python(v)) @@ -113,11 +123,11 @@ class DeReference(object): """Fetch all references and convert to their document objects """ object_map = {} - for col, dbrefs in self.reference_map.iteritems(): + for collection, dbrefs in self.reference_map.iteritems(): keys = object_map.keys() refs = list(set([dbref for dbref in dbrefs if unicode(dbref).encode('utf-8') not in keys])) - if hasattr(col, 'objects'): # We have a document class for the refs - references = col.objects.in_bulk(refs) + if hasattr(collection, 'objects'): # We have a document class for the refs + references = collection.objects.in_bulk(refs) for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document @@ -125,19 +135,19 @@ class DeReference(object): continue if doc_type: - references = doc_type._get_db()[col].find({'_id': {'$in': refs}}) + references = doc_type._get_db()[collection].find({'_id': {'$in': refs}}) for ref in references: doc = doc_type._from_son(ref) object_map[doc.id] = doc else: - references = get_db()[col].find({'_id': {'$in': refs}}) + references = get_db()[collection].find({'_id': {'$in': refs}}) for ref in references: if '_cls' in ref: doc = get_document(ref["_cls"])._from_son(ref) elif doc_type is None: doc = get_document( ''.join(x.capitalize() - for x in col.split('_')))._from_son(ref) + for x in collection.split('_')))._from_son(ref) else: doc = doc_type._from_son(ref) object_map[doc.id] = doc From c9dc4419152fe4e25bee5e8fcfc063f9560d8615 Mon Sep 17 00:00:00 2001 From: veera Date: Mon, 5 Aug 2013 15:33:54 +0530 Subject: [PATCH 04/89] Overridden the prepare_query_value method in SequenceField inorder to return the value as the required type. --- mongoengine/fields.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 826e1258..f8830d08 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1517,6 +1517,14 @@ class SequenceField(BaseField): return super(SequenceField, self).__set__(instance, value) + def prepare_query_value(self, op, value): + """ + This method is overriden in order to convert the query value into to required + type. We need to do this in order to be able to successfully compare query + values passed as string, the base implementation returns the value as is. + """ + return self.value_decorator(value) + def to_python(self, value): if value is None: value = self.generate() From 857cd718dfeb4710ca78bc5f15e209cca6e5ca85 Mon Sep 17 00:00:00 2001 From: Eric Plumb Date: Fri, 8 Nov 2013 14:57:35 -0800 Subject: [PATCH 05/89] Fix for issue #425 - allow undeclared fields in an embedded dynamic document to be seen by queryset methods --- mongoengine/base/document.py | 13 +++++++++++-- tests/document/dynamic.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index cea2f09b..8bdee201 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -792,8 +792,17 @@ class BaseDocument(object): # Look up subfield on the previous field new_field = field.lookup_member(field_name) if not new_field and isinstance(field, ComplexBaseField): - fields.append(field_name) - continue + if hasattr(field.field, 'document_type') and cls._dynamic \ + and field.field.document_type._dynamic: + DynamicField = _import_class('DynamicField') + new_field = DynamicField(db_field=field_name) + else: + fields.append(field_name) + continue + elif not new_field and hasattr(field, 'document_type') and cls._dynamic \ + and field.document_type._dynamic: + DynamicField = _import_class('DynamicField') + new_field = DynamicField(db_field=field_name) elif not new_field: raise LookUpError('Cannot resolve field "%s"' % field_name) diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index 6263e68c..aa9ea7a8 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -292,6 +292,21 @@ class DynamicTest(unittest.TestCase): person.save() self.assertEqual(Person.objects.first().age, 35) + def test_dynamic_embedded_works_with_only(self): + """Ensure custom fieldnames on a dynamic embedded document are found by qs.only()""" + + class Address(DynamicEmbeddedDocument): + city = StringField() + + class Person(DynamicDocument): + address = EmbeddedDocumentField(Address) + + Person.drop_collection() + + Person(name="Eric", address=Address(city="San Francisco", street_number="1337")).save() + + self.assertEqual(Person.objects.first().address.street_number, '1337') + self.assertEqual(Person.objects.only('address.street_number').first().address.street_number, '1337') if __name__ == '__main__': unittest.main() From 0d4afad342b6412d322c3ef20c61c7b51e9698ef Mon Sep 17 00:00:00 2001 From: Andrei Zbikowski Date: Fri, 24 Jan 2014 16:54:29 -0600 Subject: [PATCH 06/89] Fixes issue with recursive embedded document errors --- mongoengine/base/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index f5eae8ff..c625f8b8 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -317,7 +317,7 @@ class BaseDocument(object): pk = "None" if hasattr(self, 'pk'): pk = self.pk - elif self._instance: + elif self._instance and hasattr(self._instance, 'pk'): pk = self._instance.pk message = "ValidationError (%s:%s) " % (self._class_name, pk) raise ValidationError(message, errors=errors) From b085993901e57797bdf8944e86a8a2611ac1e823 Mon Sep 17 00:00:00 2001 From: "Brian J. Dowling" Date: Mon, 27 Jan 2014 23:05:29 +0000 Subject: [PATCH 07/89] Allow dynamic dictionary-style field access Allows the doc[key] syntax to work for dynamicembeddeddocument fields Fixes #559 --- mongoengine/base/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index f5eae8ff..c5ef087c 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -182,7 +182,7 @@ class BaseDocument(object): """Dictionary-style field access, set a field's value. """ # Ensure that the field exists before settings its value - if name not in self._fields: + if not self._dynamic and name not in self._fields: raise KeyError(name) return setattr(self, name, value) From 9b2080d036859c4456fc56c3085a14e791853035 Mon Sep 17 00:00:00 2001 From: "Brian J. Dowling" Date: Tue, 28 Jan 2014 22:10:26 -0500 Subject: [PATCH 08/89] Added a test for allowing dynamic dictionary-style field access Closes #559 --- tests/document/dynamic.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index 6263e68c..bf69cb27 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -292,6 +292,44 @@ class DynamicTest(unittest.TestCase): person.save() self.assertEqual(Person.objects.first().age, 35) + def test_dynamic_and_embedded_dict_access(self): + """Ensure embedded dynamic documents work with dict[] style access""" + + class Address(EmbeddedDocument): + city = StringField() + + class Person(DynamicDocument): + name = StringField() + + Person.drop_collection() + + Person(name="Ross", address=Address(city="London")).save() + + person = Person.objects.first() + person.attrval = "This works" + + person["phone"] = "555-1212" # but this should too + + # Same thing two levels deep + person["address"]["city"] = "Lundenne" + person.save() + + self.assertEqual(Person.objects.first().address.city, "Lundenne") + + self.assertEqual(Person.objects.first().phone, "555-1212") + + person = Person.objects.first() + person.address = Address(city="Londinium") + person.save() + + self.assertEqual(Person.objects.first().address.city, "Londinium") + + + person = Person.objects.first() + person["age"] = 35 + person.save() + self.assertEqual(Person.objects.first().age, 35) + if __name__ == '__main__': unittest.main() From c5c7378c63a7cf12223d1778a66f262f6c7eff6a Mon Sep 17 00:00:00 2001 From: tprimozi Date: Tue, 4 Feb 2014 13:41:17 +0000 Subject: [PATCH 09/89] Implemented equality between Documents and DBRefs --- mongoengine/base/document.py | 5 +++-- mongoengine/document.py | 2 +- tests/document/instance.py | 26 ++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index f5eae8ff..a7666efd 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -214,8 +214,9 @@ class BaseDocument(object): def __eq__(self, other): if isinstance(other, self.__class__) and hasattr(other, 'id'): - if self.id == other.id: - return True + return self.id == other.id + if isinstance(other, DBRef): + return self.id == other.id return False def __ne__(self, other): diff --git a/mongoengine/document.py b/mongoengine/document.py index 114778eb..60ecfe9e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -67,7 +67,7 @@ class EmbeddedDocument(BaseDocument): def __eq__(self, other): if isinstance(other, self.__class__): - return self.to_mongo() == other.to_mongo() + return self._data == other._data return False def __ne__(self, other): diff --git a/tests/document/instance.py b/tests/document/instance.py index 07db85a0..a842a221 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2452,5 +2452,31 @@ class InstanceTest(unittest.TestCase): f1.ref # Dereferences lazily self.assertEqual(f1, f2) + def test_dbref_equality(self): + class Test2(Document): + name = StringField() + + class Test(Document): + name = StringField() + test2 = ReferenceField('Test2') + + Test.drop_collection() + Test2.drop_collection() + + t2 = Test2(name='a') + t2.save() + + t = Test(name='b', test2 = t2) + + f = Test._from_son(t.to_mongo()) + + dbref = f._data['test2'] + obj = f.test2 + self.assertTrue(isinstance(dbref, DBRef)) + self.assertTrue(isinstance(obj, Test2)) + self.assertTrue(obj.id == dbref.id) + self.assertTrue(obj == dbref) + self.assertTrue(dbref == obj) + if __name__ == '__main__': unittest.main() From 0523c2ea4b6d4f4788fe5c99581fa504c16eba74 Mon Sep 17 00:00:00 2001 From: tprimozi Date: Thu, 13 Feb 2014 18:12:33 +0000 Subject: [PATCH 10/89] Fixed document equality: documents in different collections can have equal ids. --- mongoengine/base/document.py | 2 +- tests/document/instance.py | 50 ++++++++++++++++++++++++++++++------ 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index a7666efd..be0635ce 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -216,7 +216,7 @@ class BaseDocument(object): if isinstance(other, self.__class__) and hasattr(other, 'id'): return self.id == other.id if isinstance(other, DBRef): - return self.id == other.id + return self._get_collection_name() == other.collection and self.id == other.id return False def __ne__(self, other): diff --git a/tests/document/instance.py b/tests/document/instance.py index a842a221..cdf2b2c1 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2456,27 +2456,61 @@ class InstanceTest(unittest.TestCase): class Test2(Document): name = StringField() + class Test3(Document): + name = StringField() + class Test(Document): name = StringField() test2 = ReferenceField('Test2') + test3 = ReferenceField('Test3') Test.drop_collection() Test2.drop_collection() + Test3.drop_collection() t2 = Test2(name='a') t2.save() - t = Test(name='b', test2 = t2) + t3 = Test3(name='x') + t3.id = t2.id + t3.save() + + t = Test(name='b', test2=t2, test3=t3) f = Test._from_son(t.to_mongo()) - dbref = f._data['test2'] - obj = f.test2 - self.assertTrue(isinstance(dbref, DBRef)) - self.assertTrue(isinstance(obj, Test2)) - self.assertTrue(obj.id == dbref.id) - self.assertTrue(obj == dbref) - self.assertTrue(dbref == obj) + dbref2 = f._data['test2'] + obj2 = f.test2 + self.assertTrue(isinstance(dbref2, DBRef)) + self.assertTrue(isinstance(obj2, Test2)) + self.assertTrue(obj2.id == dbref2.id) + self.assertTrue(obj2 == dbref2) + self.assertTrue(dbref2 == obj2) + + dbref3 = f._data['test3'] + obj3 = f.test3 + self.assertTrue(isinstance(dbref3, DBRef)) + self.assertTrue(isinstance(obj3, Test3)) + self.assertTrue(obj3.id == dbref3.id) + self.assertTrue(obj3 == dbref3) + self.assertTrue(dbref3 == obj3) + + self.assertTrue(obj2.id == obj3.id) + self.assertTrue(dbref2.id == dbref3.id) + self.assertFalse(dbref2 == dbref3) + self.assertFalse(dbref3 == dbref2) + self.assertTrue(dbref2 != dbref3) + self.assertTrue(dbref3 != dbref2) + + self.assertFalse(obj2 == dbref3) + self.assertFalse(dbref3 == obj2) + self.assertTrue(obj2 != dbref3) + self.assertTrue(dbref3 != obj2) + + self.assertFalse(obj3 == dbref2) + self.assertFalse(dbref2 == obj3) + self.assertTrue(obj3 != dbref2) + self.assertTrue(dbref2 != obj3) if __name__ == '__main__': unittest.main() From db1e69813bc0b31957e7589723a2f80cda74454b Mon Sep 17 00:00:00 2001 From: Frank Battaglia Date: Thu, 31 Oct 2013 20:43:07 -0400 Subject: [PATCH 11/89] add atomic conditions to save Conflicts: mongoengine/document.py --- mongoengine/document.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 114778eb..f870a078 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -13,7 +13,8 @@ from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList, ALLOW_INHERITANCE, get_document) from mongoengine.errors import ValidationError -from mongoengine.queryset import OperationError, NotUniqueError, QuerySet +from mongoengine.queryset import (OperationError, NotUniqueError, + QuerySet, transform) from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME from mongoengine.context_managers import switch_db, switch_collection @@ -180,7 +181,7 @@ class Document(BaseDocument): def save(self, force_insert=False, validate=True, clean=True, write_concern=None, cascade=None, cascade_kwargs=None, - _refs=None, **kwargs): + _refs=None, save_condition=None, **kwargs): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. @@ -203,7 +204,8 @@ class Document(BaseDocument): :param cascade_kwargs: (optional) kwargs dictionary to be passed throw to cascading saves. Implies ``cascade=True``. :param _refs: A list of processed references used in cascading saves - + :param save_condition: only perform save if matching record in db + satisfies condition(s) (e.g., version number) .. versionchanged:: 0.5 In existing documents it only saves changed fields using set / unset. Saves are cascaded and any @@ -217,6 +219,9 @@ class Document(BaseDocument): meta['cascade'] = True. Also you can pass different kwargs to the cascade save using cascade_kwargs which overwrites the existing kwargs with custom values. + .. versionchanged:: 0.8.5 + Optional save_condition that only overwrites existing documents + if the condition is satisfied in the current db record. """ signals.pre_save.send(self.__class__, document=self) @@ -230,7 +235,8 @@ class Document(BaseDocument): created = ('_id' not in doc or self._created or force_insert) - signals.pre_save_post_validation.send(self.__class__, document=self, created=created) + signals.pre_save_post_validation.send(self.__class__, document=self, + created=created) try: collection = self._get_collection() @@ -243,7 +249,12 @@ class Document(BaseDocument): object_id = doc['_id'] updates, removals = self._delta() # Need to add shard key to query, or you get an error - select_dict = {'_id': object_id} + if save_condition is not None: + select_dict = transform.query(self.__class__, + **save_condition) + else: + select_dict = {} + select_dict['_id'] = object_id shard_key = self.__class__._meta.get('shard_key', tuple()) for k in shard_key: actual_key = self._db_field_map.get(k, k) @@ -263,10 +274,12 @@ class Document(BaseDocument): if removals: update_query["$unset"] = removals if updates or removals: + upsert = save_condition is None last_error = collection.update(select_dict, update_query, - upsert=True, **write_concern) + upsert=upsert, **write_concern) created = is_new_object(last_error) + if cascade is None: cascade = self._meta.get('cascade', False) or cascade_kwargs is not None From 673a966541e8b2083616497378f54c021f5b0c08 Mon Sep 17 00:00:00 2001 From: Frank Battaglia Date: Tue, 12 Nov 2013 15:22:27 -0500 Subject: [PATCH 12/89] add tests for save_condition kwarg in document.save() --- tests/document/instance.py | 62 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/document/instance.py b/tests/document/instance.py index 07db85a0..4747c0ed 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -820,6 +820,68 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) + def test_save_atomicity_condition(self): + + class Widget(Document): + toggle = BooleanField(default=False) + count = IntField(default=0) + save_id = UUIDField() + + def flip(widget): + widget.toggle = not widget.toggle + widget.count += 1 + + def UUID(i): + return uuid.UUID(int=i) + + Widget.drop_collection() + + w1 = Widget(toggle=False, save_id=UUID(1)) + + # ignore save_condition on new record creation + w1.save(save_condition={'save_id':UUID(42)}) + w1.reload() + self.assertFalse(w1.toggle) + self.assertEqual(w1.save_id, UUID(1)) + self.assertEqual(w1.count, 0) + + # mismatch in save_condition prevents save + flip(w1) + self.assertTrue(w1.toggle) + self.assertEqual(w1.count, 1) + w1.save(save_condition={'save_id':UUID(42)}) + w1.reload() + self.assertFalse(w1.toggle) + self.assertEqual(w1.count, 0) + + # matched save_condition allows save + flip(w1) + self.assertTrue(w1.toggle) + self.assertEqual(w1.count, 1) + w1.save(save_condition={'save_id':UUID(1)}) + w1.reload() + self.assertTrue(w1.toggle) + self.assertEqual(w1.count, 1) + + # save_condition can be used to ensure atomic read & updates + # i.e., prevent interleaved reads and writes from separate contexts + w2 = Widget.objects.get() + self.assertEqual(w1, w2) + old_id = w1.save_id + + flip(w1) + w1.save(save_condition={'save_id':old_id}) + w1.reload() + self.assertFalse(w1.toggle) + self.assertEqual(w1.count, 2) + flip(w2) + flip(w2) + w2.save(save_condition={'save_id':old_id}) + w2.reload() + self.assertFalse(w1.toggle) + self.assertEqual(w1.count, 2) + + def test_update(self): """Ensure that an existing document is updated instead of be overwritten.""" From 0a2dbbc58b5ae0df92e5465c1a43d46b71d0adb4 Mon Sep 17 00:00:00 2001 From: Frank Battaglia Date: Tue, 12 Nov 2013 16:01:56 -0500 Subject: [PATCH 13/89] add tests for mongo query operators --- tests/document/instance.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 4747c0ed..df6b4fcb 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -870,6 +870,7 @@ class InstanceTest(unittest.TestCase): old_id = w1.save_id flip(w1) + w1.save_id = UUID(2) w1.save(save_condition={'save_id':old_id}) w1.reload() self.assertFalse(w1.toggle) @@ -878,8 +879,20 @@ class InstanceTest(unittest.TestCase): flip(w2) w2.save(save_condition={'save_id':old_id}) w2.reload() - self.assertFalse(w1.toggle) - self.assertEqual(w1.count, 2) + self.assertFalse(w2.toggle) + self.assertEqual(w2.count, 2) + + # save_condition uses mongoengine-style operator syntax + flip(w1) + w1.save(save_condition={'count__lt':w1.count}) + w1.reload() + self.assertTrue(w1.toggle) + self.assertEqual(w1.count, 3) + flip(w1) + w1.save(save_condition={'count__gte':w1.count}) + w1.reload() + self.assertTrue(w1.toggle) + self.assertEqual(w1.count, 3) def test_update(self): From 86363986fc0521262676811835469efdb1530ddb Mon Sep 17 00:00:00 2001 From: Frank Battaglia Date: Tue, 12 Nov 2013 16:37:20 -0500 Subject: [PATCH 14/89] whitespace --- tests/document/instance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index df6b4fcb..acb26c6d 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -894,7 +894,6 @@ class InstanceTest(unittest.TestCase): self.assertTrue(w1.toggle) self.assertEqual(w1.count, 3) - def test_update(self): """Ensure that an existing document is updated instead of be overwritten.""" From 9d125c9e797643fbe776114407da321560ef7ca2 Mon Sep 17 00:00:00 2001 From: Frank Battaglia Date: Sun, 23 Feb 2014 19:37:42 -0500 Subject: [PATCH 15/89] inherit parent Document type _auto_id_field value --- mongoengine/base/metaclasses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index ff5afddf..4b2e8b9b 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -359,7 +359,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class.id = field # Set primary key if not defined by the document - new_class._auto_id_field = False + new_class._auto_id_field = getattr(parent_doc_cls, + '_auto_id_field', False) if not new_class._meta.get('id_field'): new_class._auto_id_field = True new_class._meta['id_field'] = 'id' From 295ef3dc1deb1a805f8b19e3e859a381f825f119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 25 Feb 2014 15:36:30 -0300 Subject: [PATCH 16/89] db_alias support and fixes for custom map/reduce output --- mongoengine/queryset/base.py | 33 ++++++++- tests/queryset/queryset.py | 136 ++++++++++++++++++++++++++++++++++- 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c2ad027e..4bd7128e 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -7,12 +7,14 @@ import pprint import re import warnings +from bson import SON from bson.code import Code from bson import json_util import pymongo from pymongo.common import validate_read_preference from mongoengine import signals +from mongoengine.connection import get_db from mongoengine.common import _import_class from mongoengine.base.common import get_document from mongoengine.errors import (OperationError, NotUniqueError, @@ -923,7 +925,36 @@ class BaseQuerySet(object): map_reduce_function = 'inline_map_reduce' else: map_reduce_function = 'map_reduce' - mr_args['out'] = output + + if isinstance(output, basestring): + mr_args['out'] = output + + elif isinstance(output, dict): + ordered_output = [] + + for part in ('replace', 'merge', 'reduce'): + value = output.get(part) + if value: + ordered_output.append((part, value)) + break + + else: + raise OperationError("actionData not specified for output") + + db_alias = output.get('db_alias') + remaing_args = ['db', 'sharded', 'nonAtomic'] + + if db_alias: + ordered_output.append(('db', get_db(db_alias).name)) + del remaing_args[0] + + + for part in remaing_args: + value = output.get(part) + if value: + ordered_output.append((part, value)) + + mr_args['out'] = SON(ordered_output) results = getattr(queryset._collection, map_reduce_function)( map_f, reduce_f, **mr_args) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7ff2965d..2fcd466c 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -14,7 +14,7 @@ from pymongo.read_preferences import ReadPreference from bson import ObjectId from mongoengine import * -from mongoengine.connection import get_connection +from mongoengine.connection import get_connection, get_db from mongoengine.python_support import PY3 from mongoengine.context_managers import query_counter from mongoengine.queryset import (QuerySet, QuerySetManager, @@ -1925,6 +1925,140 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_map_reduce_custom_output(self): + """ + Test map/reduce custom output + """ + register_connection('test2', 'mongoenginetest2') + + class Family(Document): + id = IntField( + primary_key=True) + log = StringField() + + class Person(Document): + id = IntField( + primary_key=True) + name = StringField() + age = IntField() + family = ReferenceField(Family) + + Family.drop_collection() + Person.drop_collection() + + # creating first family + f1 = Family(id=1, log="Trav 02 de Julho") + f1.save() + + # persons of first family + Person(id=1, family=f1, name=u"Wilson Jr", age=21).save() + Person(id=2, family=f1, name=u"Wilson Father", age=45).save() + Person(id=3, family=f1, name=u"Eliana Costa", age=40).save() + Person(id=4, family=f1, name=u"Tayza Mariana", age=17).save() + + # creating second family + f2 = Family(id=2, log="Av prof frasc brunno") + f2.save() + + #persons of second family + Person(id=5, family=f2, name="Isabella Luanna", age=16).save() + Person(id=6, family=f2, name="Sandra Mara", age=36).save() + Person(id=7, family=f2, name="Igor Gabriel", age=10).save() + + # creating third family + f3 = Family(id=3, log="Av brazil") + f3.save() + + #persons of thrird family + Person(id=8, family=f3, name="Arthur WA", age=30).save() + Person(id=9, family=f3, name="Paula Leonel", age=25).save() + + # executing join map/reduce + map_person = """ + function () { + emit(this.family, { + totalAge: this.age, + persons: [{ + name: this.name, + age: this.age + }]}); + } + """ + + map_family = """ + function () { + emit(this._id, { + totalAge: 0, + persons: [] + }); + } + """ + + reduce_f = """ + function (key, values) { + var family = {persons: [], totalAge: 0}; + + values.forEach(function(value) { + if (value.persons) { + value.persons.forEach(function (person) { + family.persons.push(person); + family.totalAge += person.age; + }); + } + }); + + return family; + } + """ + cursor = Family.objects.map_reduce( + map_f=map_family, + reduce_f=reduce_f, + output={'replace': 'family_map', 'db_alias': 'test2'}) + + # start a map/reduce + cursor.next() + + results = Person.objects.map_reduce( + map_f=map_person, + reduce_f=reduce_f, + output={'reduce': 'family_map', 'db_alias': 'test2'}) + + results = list(results) + collection = get_db('test2').family_map + + self.assertEqual( + collection.find_one({'_id': 1}), { + '_id': 1, + 'value': { + 'persons': [ + {'age': 21, 'name': u'Wilson Jr'}, + {'age': 45, 'name': u'Wilson Father'}, + {'age': 40, 'name': u'Eliana Costa'}, + {'age': 17, 'name': u'Tayza Mariana'}], + 'totalAge': 123} + }) + + self.assertEqual( + collection.find_one({'_id': 2}), { + '_id': 2, + 'value': { + 'persons': [ + {'age': 16, 'name': u'Isabella Luanna'}, + {'age': 36, 'name': u'Sandra Mara'}, + {'age': 10, 'name': u'Igor Gabriel'}], + 'totalAge': 62} + }) + + self.assertEqual( + collection.find_one({'_id': 3}), { + '_id': 3, + 'value': { + 'persons': [ + {'age': 30, 'name': u'Arthur WA'}, + {'age': 25, 'name': u'Paula Leonel'}], + 'totalAge': 55} + }) + def test_map_reduce_finalize(self): """Ensure that map, reduce, and finalize run and introduce "scope" by simulating "hotness" ranking with Reddit algorithm. From ef55e6d476a89838103f70e420b6535d8186ac99 Mon Sep 17 00:00:00 2001 From: Vlad Zloteanu Date: Sat, 1 Mar 2014 17:51:59 +0100 Subject: [PATCH 17/89] fixes MongoEngine/mongoengine#589 --- mongoengine/document.py | 2 +- tests/test_signals.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 114778eb..3aaede27 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -296,9 +296,9 @@ class Document(BaseDocument): if id_field not in self._meta.get('shard_key', []): self[id_field] = self._fields[id_field].to_python(object_id) + signals.post_save.send(self.__class__, document=self, created=created) self._clear_changed_fields() self._created = False - signals.post_save.send(self.__class__, document=self, created=created) return self def cascade_save(self, *args, **kwargs): diff --git a/tests/test_signals.py b/tests/test_signals.py index 50e5e6b8..3d0cbb3e 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -54,7 +54,9 @@ class SignalTests(unittest.TestCase): @classmethod def post_save(cls, sender, document, **kwargs): + dirty_keys = document._delta()[0].keys() + document._delta()[1].keys() signal_output.append('post_save signal, %s' % document) + signal_output.append('post_save dirty keys, %s' % dirty_keys) if 'created' in kwargs: if kwargs['created']: signal_output.append('Is created') @@ -203,6 +205,7 @@ class SignalTests(unittest.TestCase): "pre_save_post_validation signal, Bill Shakespeare", "Is created", "post_save signal, Bill Shakespeare", + "post_save dirty keys, ['name']", "Is created" ]) @@ -213,6 +216,7 @@ class SignalTests(unittest.TestCase): "pre_save_post_validation signal, William Shakespeare", "Is updated", "post_save signal, William Shakespeare", + "post_save dirty keys, ['name']", "Is updated" ]) From 8bcbc6d5451aebbd934973cb3afcd08be4f36171 Mon Sep 17 00:00:00 2001 From: Brian Helba Date: Sun, 2 Mar 2014 18:35:49 -0500 Subject: [PATCH 18/89] Add authentication_source option to register_connection (#573) (#580) Since v2.5, PyMongo has supported a "source" option, to specify a particular database to authenticate against. This adds support for that option, in the form of a "authentication_source" option to register_connection. --- mongoengine/connection.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 7cc626f4..5db3e1e6 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -20,7 +20,8 @@ _dbs = {} def register_connection(alias, name, host=None, port=None, is_slave=False, read_preference=False, slaves=None, - username=None, password=None, **kwargs): + username=None, password=None, authentication_source=None, + **kwargs): """Add a connection. :param alias: the name that will be used to refer to this connection @@ -36,6 +37,7 @@ def register_connection(alias, name, host=None, port=None, be a registered connection that has :attr:`is_slave` set to ``True`` :param username: username to authenticate with :param password: password to authenticate with + :param authentication_source: database to authenticate against :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver """ @@ -46,10 +48,11 @@ def register_connection(alias, name, host=None, port=None, 'host': host or 'localhost', 'port': port or 27017, 'is_slave': is_slave, + 'read_preference': read_preference, 'slaves': slaves or [], 'username': username, 'password': password, - 'read_preference': read_preference + 'authentication_source': authentication_source } # Handle uri style connections @@ -99,6 +102,7 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn_settings.pop('is_slave', None) conn_settings.pop('username', None) conn_settings.pop('password', None) + conn_settings.pop('authentication_source', None) else: # Get all the slave connections if 'slaves' in conn_settings: @@ -137,7 +141,8 @@ def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False): # Authenticate if necessary if conn_settings['username'] and conn_settings['password']: db.authenticate(conn_settings['username'], - conn_settings['password']) + conn_settings['password'], + source=conn_settings['authentication_source']) _dbs[alias] = db return _dbs[alias] From 19314e7e06c3b4b393b0017868182c689c75f9f9 Mon Sep 17 00:00:00 2001 From: Kirill Pavlov Date: Mon, 3 Mar 2014 13:09:26 +0800 Subject: [PATCH 19/89] fix docstring for DictField --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 82642cda..b0e51a24 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -760,7 +760,7 @@ class DictField(ComplexBaseField): similar to an embedded document, but the structure is not defined. .. note:: - Required means it cannot be empty - as the default for ListFields is [] + Required means it cannot be empty - as the default for DictFields is {} .. versionadded:: 0.3 .. versionchanged:: 0.5 - Can now handle complex / varying types of data From c82f4f0d45e6e70bf8410d28415602ddea6794e3 Mon Sep 17 00:00:00 2001 From: Phil Freo Date: Fri, 7 Mar 2014 13:29:29 -0800 Subject: [PATCH 20/89] clarifying the 'push' atomic update docs the first time I read this I was all like... "no duh it will remove either the first or the last, but which does it do???" --- docs/guide/querying.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 32cbb94e..7168b5e8 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -499,11 +499,13 @@ that you may use with these methods: * ``dec`` -- decrement a value by a given amount * ``push`` -- append a value to a list * ``push_all`` -- append several values to a list -* ``pop`` -- remove the first or last element of a list +* ``pop`` -- remove the first or last element of a list `depending on the value`_ * ``pull`` -- remove a value from a list * ``pull_all`` -- remove several values from a list * ``add_to_set`` -- add value to a list only if its not in the list already +.. _depending on the value: http://docs.mongodb.org/manual/reference/operator/update/pop/ + The syntax for atomic updates is similar to the querying syntax, but the modifier comes before the field, not after it:: From 5d9ec0b20854eac476f974b495a1d444c7134bf8 Mon Sep 17 00:00:00 2001 From: Nicolas Despres Date: Mon, 17 Mar 2014 17:19:17 +0100 Subject: [PATCH 21/89] Save is called on the document not the file field. --- docs/guide/gridfs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 596585de..68e7a6d2 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -46,7 +46,7 @@ slightly different manner. First, a new file must be created by calling the marmot.photo.write('some_more_image_data') marmot.photo.close() - marmot.photo.save() + marmot.save() Deletion -------- From c1f88a4e14f319b8a849dad44849d6f6884c8e75 Mon Sep 17 00:00:00 2001 From: Falcon Dai Date: Mon, 17 Mar 2014 22:29:53 -0500 Subject: [PATCH 22/89] minor change to geo-related docs --- docs/guide/defining-documents.rst | 2 ++ mongoengine/fields.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 5d8b628a..07bce3bb 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -531,6 +531,8 @@ field name to the index definition. Sometimes its more efficient to index parts of Embedded / dictionary fields, in this case use 'dot' notation to identify the value to index eg: `rank.title` +.. _geospatial-indexes: + Geospatial indexes ------------------ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 82642cda..48db6dfd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1613,7 +1613,12 @@ class UUIDField(BaseField): class GeoPointField(BaseField): - """A list storing a latitude and longitude. + """A list storing a longitude and latitude coordinate. + + .. note:: this represents a generic point in a 2D plane and a legacy way of + representing a geo point. It admits 2d indexes but not "2dsphere" indexes + in MongoDB > 2.4 which are more natural for modeling geospatial points. + See :ref:`geospatial-indexes` .. versionadded:: 0.4 """ @@ -1635,7 +1640,7 @@ class GeoPointField(BaseField): class PointField(GeoJsonBaseField): - """A geo json field storing a latitude and longitude. + """A GeoJSON field storing a longitude and latitude coordinate. The data is represented as: @@ -1654,7 +1659,7 @@ class PointField(GeoJsonBaseField): class LineStringField(GeoJsonBaseField): - """A geo json field storing a line of latitude and longitude coordinates. + """A GeoJSON field storing a line of longitude and latitude coordinates. The data is represented as: @@ -1672,7 +1677,7 @@ class LineStringField(GeoJsonBaseField): class PolygonField(GeoJsonBaseField): - """A geo json field storing a polygon of latitude and longitude coordinates. + """A GeoJSON field storing a polygon of longitude and latitude coordinates. The data is represented as: From 4d7b988018c2d150401fb96db36da1ebdd6a2c4d Mon Sep 17 00:00:00 2001 From: poly Date: Tue, 1 Apr 2014 19:52:21 +0800 Subject: [PATCH 23/89] Fixed uncorrectly split a query key, when it ends with "_" --- mongoengine/queryset/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index e31a8b7d..27e41ad2 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -38,7 +38,7 @@ def query(_doc_cls=None, _field_operation=False, **query): mongo_query.update(value) continue - parts = key.split('__') + parts = key.rsplit('__') indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is From 803caddbd40ab2ea6cbee8e6b65a5921277fcafc Mon Sep 17 00:00:00 2001 From: Dmitry Konishchev Date: Wed, 9 Apr 2014 14:25:53 +0400 Subject: [PATCH 24/89] Raise NotUniqueError in Document.update() on pymongo.errors.DuplicateKeyError --- mongoengine/queryset/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c2ad027e..c34c1479 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -443,6 +443,8 @@ class BaseQuerySet(object): return result elif result: return result['n'] + except pymongo.errors.DuplicateKeyError, err: + raise NotUniqueError(u'Update failed (%s)' % unicode(err)) except pymongo.errors.OperationFailure, err: if unicode(err) == u'multi not coded yet': message = u'update() method requires MongoDB 1.1.3+' From b45a601ad2a208b209af5e8f45893aaf16ebd59f Mon Sep 17 00:00:00 2001 From: Dmitry Konishchev Date: Tue, 15 Apr 2014 19:32:42 +0400 Subject: [PATCH 25/89] Test raising NotUniqueError by Document.update() --- tests/document/instance.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 07db85a0..c57102da 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -15,7 +15,7 @@ from tests.fixtures import (PickleEmbedded, PickleTest, PickleSignalsTest, from mongoengine import * from mongoengine.errors import (NotRegistered, InvalidDocumentError, - InvalidQueryError) + InvalidQueryError, NotUniqueError) from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db from mongoengine.base import get_document @@ -990,6 +990,16 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidQueryError, update_no_op_raises) + def test_update_unique_field(self): + class Doc(Document): + name = StringField(unique=True) + + doc1 = Doc(name="first").save() + doc2 = Doc(name="second").save() + + self.assertRaises(NotUniqueError, lambda: + doc2.update(set__name=doc1.name)) + def test_embedded_update(self): """ Test update on `EmbeddedDocumentField` fields From 12809ebc741dc04677a4e3b0f7c3ed5c23563d7f Mon Sep 17 00:00:00 2001 From: Jatin Chopra Date: Tue, 6 May 2014 00:25:55 -0700 Subject: [PATCH 26/89] Updated Jatin's name and github name --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index d6994d50..5fa0cb3d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -171,7 +171,7 @@ that much better: * Michael Bartnett (https://github.com/michaelbartnett) * Alon Horev (https://github.com/alonho) * Kelvin Hammond (https://github.com/kelvinhammond) - * Jatin- (https://github.com/jatin-) + * Jatin Chopra (https://github.com/jatin) * Paul Uithol (https://github.com/PaulUithol) * Thom Knowles (https://github.com/fleat) * Paul (https://github.com/squamous) From babbc8bcd64893a328f34d1cde17dd7125b6dd2f Mon Sep 17 00:00:00 2001 From: Ronald van Rij Date: Mon, 5 May 2014 16:39:55 +0200 Subject: [PATCH 27/89] When using autogenerated document ids in a sharded collection, do set that id back into the Document --- mongoengine/document.py | 2 +- tests/document/instance.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 114778eb..e09503c8 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -293,7 +293,7 @@ class Document(BaseDocument): raise NotUniqueError(message % unicode(err)) raise OperationError(message % unicode(err)) id_field = self._meta['id_field'] - if id_field not in self._meta.get('shard_key', []): + if created or id_field not in self._meta.get('shard_key', []): self[id_field] = self._fields[id_field].to_python(object_id) self._clear_changed_fields() diff --git a/tests/document/instance.py b/tests/document/instance.py index 07db85a0..14cc0074 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2281,6 +2281,8 @@ class InstanceTest(unittest.TestCase): log.machine = "Localhost" log.save() + self.assertIsNotNone(log.id) + log.log = "Saving" log.save() @@ -2304,6 +2306,8 @@ class InstanceTest(unittest.TestCase): log.machine = "Localhost" log.save() + self.assertIsNotNone(log.id) + log.log = "Saving" log.save() From 9544b7d9689bb2ef3672aa230b3829400be16414 Mon Sep 17 00:00:00 2001 From: Ronald van Rij Date: Fri, 9 May 2014 14:33:18 +0200 Subject: [PATCH 28/89] Fixed unit test which used assertIsNotNone --- tests/document/instance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 14cc0074..7db452d9 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2281,7 +2281,7 @@ class InstanceTest(unittest.TestCase): log.machine = "Localhost" log.save() - self.assertIsNotNone(log.id) + self.assertTrue(log.id is not None) log.log = "Saving" log.save() @@ -2306,7 +2306,7 @@ class InstanceTest(unittest.TestCase): log.machine = "Localhost" log.save() - self.assertIsNotNone(log.id) + self.assertTrue(log.id is not None) log.log = "Saving" log.save() From abcacc82f359d37922dc6c19c6b860b4c633e8e0 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Wed, 21 May 2014 22:21:46 -0700 Subject: [PATCH 29/89] dont use a system collection --- tests/document/instance.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 07db85a0..6c4da162 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2411,7 +2411,7 @@ class InstanceTest(unittest.TestCase): for parameter_name, parameter in self.parameters.iteritems(): parameter.expand() - class System(Document): + class NodesSystem(Document): name = StringField(required=True) nodes = MapField(ReferenceField(Node, dbref=False)) @@ -2419,18 +2419,18 @@ class InstanceTest(unittest.TestCase): for node_name, node in self.nodes.iteritems(): node.expand() node.save(*args, **kwargs) - super(System, self).save(*args, **kwargs) + super(NodesSystem, self).save(*args, **kwargs) - System.drop_collection() + NodesSystem.drop_collection() Node.drop_collection() - system = System(name="system") + system = NodesSystem(name="system") system.nodes["node"] = Node() system.save() system.nodes["node"].parameters["param"] = Parameter() system.save() - system = System.objects.first() + system = NodesSystem.objects.first() self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value) def test_embedded_document_equality(self): From 3faf3c84be55f709d828ea7837db988c56b0239e Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Tue, 27 May 2014 16:33:38 -0300 Subject: [PATCH 30/89] Avoid to open all documents from cursors in an if stmt Using a cursos in an if statement: cursor = Collection.objects if cursor: (...) Will open all documents, because there are not an __nonzero__ method. This change check only one document (if present) and returns True or False. --- mongoengine/queryset/base.py | 9 +++++++++ tests/queryset/queryset.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c2ad027e..823bc164 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -154,6 +154,15 @@ class BaseQuerySet(object): def __iter__(self): raise NotImplementedError + def __nonzero__(self): + """ Avoid to open all records in an if stmt """ + + for value in self: + self.rewind() + return True + + return False + # Core functions def all(self): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7ff2965d..f274e0ee 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3814,6 +3814,29 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(Example.objects(size=instance_size).count(), 1) self.assertEqual(Example.objects(size__in=[instance_size]).count(), 1) + def test_cursor_in_an_if_stmt(self): + + class Test(Document): + test_field = StringField() + + Test.drop_collection() + queryset = Test.objects + + if queryset: + raise AssertionError('Empty cursor returns True') + + test = Test() + test.test_field = 'test' + test.save() + + queryset = Test.objects + if not test: + raise AssertionError('There is data, but cursor returned False') + + queryset.next() + if queryset: + raise AssertionError('There is no data left in cursor') + if __name__ == '__main__': unittest.main() From 9ba657797ed64d50a876c95bc2a243bed3037e19 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 08:33:22 -0300 Subject: [PATCH 31/89] Authors updated according guideline --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index d6994d50..5d6a45fa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -189,3 +189,4 @@ that much better: * Tom (https://github.com/tomprimozic) * j0hnsmith (https://github.com/j0hnsmith) * Damien Churchill (https://github.com/damoxc) + * Jonathan Simon Prates (https://github.com/jonathansp) \ No newline at end of file From 47f0de9836a7e340047c6639dee9ade29b9755cb Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 08:36:57 -0300 Subject: [PATCH 32/89] Py3 fix --- mongoengine/queryset/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 823bc164..022b7c1f 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -161,7 +161,12 @@ class BaseQuerySet(object): self.rewind() return True - return False + return False + + def __bool__(self): + """ Same behaviour in Py3 """ + + return self.__nonzero__(): # Core functions From edcdfeb0573f253227a196b3080344e849f48109 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 09:03:12 -0300 Subject: [PATCH 33/89] Fix syntax error --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 022b7c1f..0dfff7cc 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -166,7 +166,7 @@ class BaseQuerySet(object): def __bool__(self): """ Same behaviour in Py3 """ - return self.__nonzero__(): + return self.__nonzero__() # Core functions From dfdecef8e72fdd0379cdec4452f3175f0826b892 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 09:40:22 -0300 Subject: [PATCH 34/89] Fix py2 and py3 --- mongoengine/queryset/base.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 0dfff7cc..09fb5bf6 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -154,7 +154,7 @@ class BaseQuerySet(object): def __iter__(self): raise NotImplementedError - def __nonzero__(self): + def _bool(self): """ Avoid to open all records in an if stmt """ for value in self: @@ -163,10 +163,15 @@ class BaseQuerySet(object): return False + def __nonzero__(self): + """ Same behaviour in Py2 """ + + return self._bool() + def __bool__(self): """ Same behaviour in Py3 """ - return self.__nonzero__() + return self._bool() # Core functions From ee0c7fd8bfa08ec885f379b2aa6dd72f48d8d7c5 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 13:21:00 -0300 Subject: [PATCH 35/89] Change for loop to self.first() --- mongoengine/queryset/base.py | 6 +----- tests/queryset/queryset.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 09fb5bf6..099831fe 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -157,11 +157,7 @@ class BaseQuerySet(object): def _bool(self): """ Avoid to open all records in an if stmt """ - for value in self: - self.rewind() - return True - - return False + return False if self.first() is None else True def __nonzero__(self): """ Same behaviour in Py2 """ diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index f274e0ee..f6adaf39 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3834,8 +3834,8 @@ class QuerySetTest(unittest.TestCase): raise AssertionError('There is data, but cursor returned False') queryset.next() - if queryset: - raise AssertionError('There is no data left in cursor') + if not queryset: + raise AssertionError('There is data, cursor must return True') if __name__ == '__main__': From 30964f65e465f7e29960cd49caf29d5c9d4ac756 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 17:06:15 -0300 Subject: [PATCH 36/89] Remove orderby in if stmt --- mongoengine/queryset/base.py | 8 +++++- tests/queryset/queryset.py | 53 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 099831fe..fe1285c6 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -157,7 +157,13 @@ class BaseQuerySet(object): def _bool(self): """ Avoid to open all records in an if stmt """ - return False if self.first() is None else True + queryset = self.clone() + queryset._ordering = [] + try: + queryset[0] + return True + except IndexError: + return False def __nonzero__(self): """ Same behaviour in Py2 """ diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index f6adaf39..65f7255b 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3837,6 +3837,59 @@ class QuerySetTest(unittest.TestCase): if not queryset: raise AssertionError('There is data, cursor must return True') + def test_bool_performance(self): + + class Person(Document): + name = StringField() + + Person.drop_collection() + for i in xrange(100): + Person(name="No: %s" % i).save() + + with query_counter() as q: + if Person.objects: + pass + + self.assertEqual(q, 1) + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertEqual(op['nreturned'], 1) + + + def test_bool_with_ordering(self): + + class Person(Document): + name = StringField() + + Person.drop_collection() + Person(name="Test").save() + + qs = Person.objects.order_by('name') + + with query_counter() as q: + + if qs: + pass + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertFalse('$orderby' in op['query'], + 'BaseQuerySet cannot use orderby in if stmt') + + with query_counter() as p: + + for x in qs: + pass + + op = p.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + + self.assertTrue('$orderby' in op['query'], + 'BaseQuerySet cannot remove orderby in for loop') + if __name__ == '__main__': unittest.main() From 39735594bd935f3003d85009c3dc14a8d3d71f23 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 17:15:48 -0300 Subject: [PATCH 37/89] Removed blank line --- tests/queryset/queryset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 65f7255b..c5fea003 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3886,7 +3886,6 @@ class QuerySetTest(unittest.TestCase): op = p.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] - self.assertTrue('$orderby' in op['query'], 'BaseQuerySet cannot remove orderby in for loop') From c87801f0a911bd02f4b4ae910751c8b04717fd02 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 17:26:28 -0300 Subject: [PATCH 38/89] Using first() from cloned queryset --- mongoengine/queryset/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index fe1285c6..85d2dc3f 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -159,11 +159,7 @@ class BaseQuerySet(object): queryset = self.clone() queryset._ordering = [] - try: - queryset[0] - return True - except IndexError: - return False + return False if queryset.first() is None else True def __nonzero__(self): """ Same behaviour in Py2 """ From c744104a18829f7f34d491a580914b89ecf3c620 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Thu, 29 May 2014 10:53:20 -0300 Subject: [PATCH 39/89] Added test with meta --- tests/queryset/queryset.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index c5fea003..bad0d1e5 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3889,6 +3889,33 @@ class QuerySetTest(unittest.TestCase): self.assertTrue('$orderby' in op['query'], 'BaseQuerySet cannot remove orderby in for loop') + def test_bool_with_ordering_from_meta_dict(self): + + class Person(Document): + name = StringField() + meta = { + 'ordering': ['name'] + } + + Person.drop_collection() + + Person(name="B").save() + Person(name="C").save() + Person(name="A").save() + + with query_counter() as q: + + if Person.objects: + pass + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' in op['query'], + 'BaseQuerySet cannot remove orderby from meta in boolen test') + + self.assertEqual(Person.objects.first().name, 'A') + if __name__ == '__main__': unittest.main() From 819ff2a90286030740e2d6d886c2f14e29d0ccc2 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Thu, 29 May 2014 14:36:30 -0300 Subject: [PATCH 40/89] Renamed to has_data() --- AUTHORS | 3 ++- mongoengine/queryset/base.py | 12 ++++++------ tests/queryset/queryset.py | 8 ++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/AUTHORS b/AUTHORS index 5d6a45fa..7b466d6b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -189,4 +189,5 @@ that much better: * Tom (https://github.com/tomprimozic) * j0hnsmith (https://github.com/j0hnsmith) * Damien Churchill (https://github.com/damoxc) - * Jonathan Simon Prates (https://github.com/jonathansp) \ No newline at end of file + * Jonathan Simon Prates (https://github.com/jonathansp) + * Thiago Papageorgiou (https://github.com/tmpapageorgiou) \ No newline at end of file diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 85d2dc3f..ef8cd2a7 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -154,22 +154,22 @@ class BaseQuerySet(object): def __iter__(self): raise NotImplementedError - def _bool(self): - """ Avoid to open all records in an if stmt """ + def has_data(self): + """ Retrieves whether cursor has any data. """ queryset = self.clone() queryset._ordering = [] return False if queryset.first() is None else True def __nonzero__(self): - """ Same behaviour in Py2 """ + """ Avoid to open all records in an if stmt in Py2. """ - return self._bool() + return self.has_data() def __bool__(self): - """ Same behaviour in Py3 """ + """ Avoid to open all records in an if stmt in Py3. """ - return self._bool() + return self.has_data() # Core functions diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index bad0d1e5..fe932367 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3831,11 +3831,15 @@ class QuerySetTest(unittest.TestCase): queryset = Test.objects if not test: - raise AssertionError('There is data, but cursor returned False') + raise AssertionError('Cursor has data and returned False') queryset.next() if not queryset: - raise AssertionError('There is data, cursor must return True') + raise AssertionError('Cursor has data and it must returns False,' + ' even in the last item.') + + self.assertTrue(queryset.has_data(), 'Cursor has data and ' + 'returned False') def test_bool_performance(self): From 85187239b629772ba276aaee54eb678c64ad8207 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Thu, 29 May 2014 15:21:24 -0300 Subject: [PATCH 41/89] Fix tests msg --- tests/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index fe932367..f68468ff 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3835,7 +3835,7 @@ class QuerySetTest(unittest.TestCase): queryset.next() if not queryset: - raise AssertionError('Cursor has data and it must returns False,' + raise AssertionError('Cursor has data and it must returns True,' ' even in the last item.') self.assertTrue(queryset.has_data(), 'Cursor has data and ' From 1eacc6fbff0bbe14a18d62032fe0b616194c8d26 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Fri, 30 May 2014 15:08:03 -0700 Subject: [PATCH 42/89] clear ordering via empty order_by --- mongoengine/queryset/base.py | 7 ++-- tests/queryset/queryset.py | 70 ++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c2ad027e..4f451667 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -50,7 +50,7 @@ class BaseQuerySet(object): self._initial_query = {} self._where_clause = None self._loaded_fields = QueryFieldList() - self._ordering = [] + self._ordering = None self._snapshot = False self._timeout = True self._class_check = True @@ -1189,8 +1189,9 @@ class BaseQuerySet(object): if self._ordering: # Apply query ordering self._cursor_obj.sort(self._ordering) - elif self._document._meta['ordering']: - # Otherwise, apply the ordering from the document model + elif self._ordering is None and self._document._meta['ordering']: + # Otherwise, apply the ordering from the document model, unless + # it's been explicitly cleared via order_by with no arguments order = self._get_order_by(self._document._meta['ordering']) self._cursor_obj.sort(order) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7ff2965d..d706eda8 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -1040,6 +1040,76 @@ class QuerySetTest(unittest.TestCase): expected = [blog_post_1, blog_post_2, blog_post_3] self.assertSequence(qs, expected) + def test_clear_ordering(self): + """ Make sure one can clear the query set ordering by applying a + consecutive order_by() + """ + + class Person(Document): + name = StringField() + + Person.drop_collection() + Person(name="A").save() + Person(name="B").save() + + qs = Person.objects.order_by('-name') + + # Make sure we can clear a previously specified ordering + with query_counter() as q: + lst = list(qs.order_by()) + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' not in op['query']) + self.assertEqual(lst[0].name, 'A') + + # Make sure previously specified ordering is preserved during + # consecutive calls to the same query set + with query_counter() as q: + lst = list(qs) + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' in op['query']) + self.assertEqual(lst[0].name, 'B') + + def test_clear_default_ordering(self): + + class Person(Document): + name = StringField() + meta = { + 'ordering': ['-name'] + } + + Person.drop_collection() + Person(name="A").save() + Person(name="B").save() + + qs = Person.objects + + # Make sure clearing default ordering works + with query_counter() as q: + lst = list(qs.order_by()) + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' not in op['query']) + self.assertEqual(lst[0].name, 'A') + + # Make sure default ordering is preserved during consecutive calls + # to the same query set + with query_counter() as q: + lst = list(qs) + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' in op['query']) + self.assertEqual(lst[0].name, 'B') + def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. """ From 9835b382dab4ed3ceef277a2948dde103542303c Mon Sep 17 00:00:00 2001 From: Sagiv Malihi Date: Thu, 3 Apr 2014 12:38:33 +0300 Subject: [PATCH 43/89] added __slots__ to BaseDocument and Document changed the _data field to static key-value mapping instead of hash table This implements #624 --- mongoengine/base/datastructures.py | 97 ++++++++++++++++++++++++++ mongoengine/base/document.py | 46 +++++++++---- mongoengine/document.py | 7 +- tests/test_datastructures.py | 107 +++++++++++++++++++++++++++++ 4 files changed, 243 insertions(+), 14 deletions(-) create mode 100644 tests/test_datastructures.py diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 4652fb56..32a66018 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -1,4 +1,6 @@ import weakref +import functools +import itertools from mongoengine.common import _import_class __all__ = ("BaseDict", "BaseList") @@ -156,3 +158,98 @@ class BaseList(list): def _mark_as_changed(self): if hasattr(self._instance, '_mark_as_changed'): self._instance._mark_as_changed(self._name) + + +class StrictDict(object): + __slots__ = () + _special_fields = set(['get', 'pop', 'iteritems', 'items', 'keys', 'create']) + _classes = {} + def __init__(self, **kwargs): + for k,v in kwargs.iteritems(): + setattr(self, k, v) + def __getitem__(self, key): + key = '_reserved_' + key if key in self._special_fields else key + try: + return getattr(self, key) + except AttributeError: + raise KeyError(key) + def __setitem__(self, key, value): + key = '_reserved_' + key if key in self._special_fields else key + return setattr(self, key, value) + def __contains__(self, key): + return hasattr(self, key) + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + def pop(self, key, default=None): + v = self.get(key, default) + try: + delattr(self, key) + except AttributeError: + pass + return v + def iteritems(self): + for key in self: + yield key, self[key] + def items(self): + return [(k, self[k]) for k in iter(self)] + def keys(self): + return list(iter(self)) + def __iter__(self): + return (key for key in self.__slots__ if hasattr(self, key)) + def __len__(self): + return len(list(self.iteritems())) + def __eq__(self, other): + return self.items() == other.items() + def __neq__(self, other): + return self.items() != other.items() + + @classmethod + def create(cls, allowed_keys): + allowed_keys_tuple = tuple(('_reserved_' + k if k in cls._special_fields else k) for k in allowed_keys) + allowed_keys = frozenset(allowed_keys_tuple) + if allowed_keys not in cls._classes: + class SpecificStrictDict(cls): + __slots__ = allowed_keys_tuple + cls._classes[allowed_keys] = SpecificStrictDict + return cls._classes[allowed_keys] + + +class SemiStrictDict(StrictDict): + __slots__ = ('_extras') + _classes = {} + def __getattr__(self, attr): + try: + super(SemiStrictDict, self).__getattr__(attr) + except AttributeError: + try: + return self.__getattribute__('_extras')[attr] + except KeyError as e: + raise AttributeError(e) + def __setattr__(self, attr, value): + try: + super(SemiStrictDict, self).__setattr__(attr, value) + except AttributeError: + try: + self._extras[attr] = value + except AttributeError: + self._extras = {attr: value} + + def __delattr__(self, attr): + try: + super(SemiStrictDict, self).__delattr__(attr) + except AttributeError: + try: + del self._extras[attr] + except KeyError as e: + raise AttributeError(e) + + def __iter__(self): + try: + extras_iter = iter(self.__getattribute__('_extras')) + except AttributeError: + extras_iter = () + return itertools.chain(super(SemiStrictDict, self).__iter__(), extras_iter) + diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index cea2f09b..01809aa9 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -16,20 +16,20 @@ from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, to_str_keys_recursive) from mongoengine.base.common import get_document, ALLOW_INHERITANCE -from mongoengine.base.datastructures import BaseDict, BaseList +from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict, SemiStrictDict from mongoengine.base.fields import ComplexBaseField __all__ = ('BaseDocument', 'NON_FIELD_ERRORS') NON_FIELD_ERRORS = '__all__' - class BaseDocument(object): + __slots__ = ('_changed_fields', '_initialised', '_created', '_data', + '_dynamic_fields', '_auto_id_field', '_db_field_map', '_cls', '__weakref__') _dynamic = False - _created = True _dynamic_lock = True - _initialised = False + STRICT = False def __init__(self, *args, **values): """ @@ -38,6 +38,8 @@ class BaseDocument(object): :param __auto_convert: Try and will cast python objects to Object types :param values: A dictionary of values for the document """ + self._initialised = False + self._created = True if args: # Combine positional arguments with named arguments. # We only want named arguments. @@ -52,8 +54,12 @@ class BaseDocument(object): values[name] = value __auto_convert = values.pop("__auto_convert", True) signals.pre_init.send(self.__class__, document=self, values=values) - - self._data = {} + + if self.STRICT and not self._dynamic: + self._data = StrictDict.create(allowed_keys=self._fields.keys())() + else: + self._data = SemiStrictDict.create(allowed_keys=self._fields.keys())() + self._dynamic_fields = SON() # Assign default values to instance @@ -129,17 +135,25 @@ class BaseDocument(object): self._data[name] = value if hasattr(self, '_changed_fields'): self._mark_as_changed(name) + try: + self__created = self._created + except AttributeError: + self__created = True - if (self._is_document and not self._created and + if (self._is_document and not self__created and name in self._meta.get('shard_key', tuple()) and self._data.get(name) != value): OperationError = _import_class('OperationError') msg = "Shard Keys are immutable. Tried to update %s" % name raise OperationError(msg) + try: + self__initialised = self._initialised + except AttributeError: + self__initialised = False # Check if the user has created a new instance of a class - if (self._is_document and self._initialised - and self._created and name == self._meta['id_field']): + if (self._is_document and self__initialised + and self__created and name == self._meta['id_field']): super(BaseDocument, self).__setattr__('_created', False) super(BaseDocument, self).__setattr__(name, value) @@ -157,9 +171,11 @@ class BaseDocument(object): if isinstance(data["_data"], SON): data["_data"] = self.__class__._from_son(data["_data"])._data for k in ('_changed_fields', '_initialised', '_created', '_data', - '_fields_ordered', '_dynamic_fields'): + '_dynamic_fields'): if k in data: setattr(self, k, data[k]) + if '_fields_ordered' in data: + setattr(type(self), '_fields_ordered', data['_fields_ordered']) dynamic_fields = data.get('_dynamic_fields') or SON() for k in dynamic_fields.keys(): setattr(self, k, data["_data"].get(k)) @@ -576,7 +592,9 @@ class BaseDocument(object): msg = ("Invalid data to create a `%s` instance.\n%s" % (cls._class_name, errors)) raise InvalidDocumentError(msg) - + + if cls.STRICT: + data = dict((k, v) for k,v in data.iteritems() if k in cls._fields) obj = cls(__auto_convert=False, **data) obj._changed_fields = changed_fields obj._created = False @@ -813,7 +831,11 @@ class BaseDocument(object): """Dynamically set the display value for a field with choices""" for attr_name, field in self._fields.items(): if field.choices: - setattr(self, + if self._dynamic: + obj = self + else: + obj = type(self) + setattr(obj, 'get_%s_display' % attr_name, partial(self.__get_field_display, field=field)) diff --git a/mongoengine/document.py b/mongoengine/document.py index 1bbd7b73..98e1d2a3 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -52,16 +52,17 @@ class EmbeddedDocument(BaseDocument): `_cls` set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` dictionary. """ + + __slots__ = ('_instance') # The __metaclass__ attribute is removed by 2to3 when running with Python3 # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 my_metaclass = DocumentMetaclass __metaclass__ = DocumentMetaclass - _instance = None - def __init__(self, *args, **kwargs): super(EmbeddedDocument, self).__init__(*args, **kwargs) + self._instance = None self._changed_fields = [] def __eq__(self, other): @@ -124,6 +125,8 @@ class Document(BaseDocument): my_metaclass = TopLevelDocumentMetaclass __metaclass__ = TopLevelDocumentMetaclass + __slots__ = ('__objects' ) + def pk(): """Primary key alias """ diff --git a/tests/test_datastructures.py b/tests/test_datastructures.py new file mode 100644 index 00000000..c761a41e --- /dev/null +++ b/tests/test_datastructures.py @@ -0,0 +1,107 @@ +import unittest +from mongoengine.base.datastructures import StrictDict, SemiStrictDict + +class TestStrictDict(unittest.TestCase): + def strict_dict_class(self, *args, **kwargs): + return StrictDict.create(*args, **kwargs) + def setUp(self): + self.dtype = self.strict_dict_class(("a", "b", "c")) + def test_init(self): + d = self.dtype(a=1, b=1, c=1) + self.assertEqual((d.a, d.b, d.c), (1, 1, 1)) + + def test_init_fails_on_nonexisting_attrs(self): + self.assertRaises(AttributeError, lambda: self.dtype(a=1, b=2, d=3)) + + def test_eq(self): + d = self.dtype(a=1, b=1, c=1) + dd = self.dtype(a=1, b=1, c=1) + e = self.dtype(a=1, b=1, c=3) + f = self.dtype(a=1, b=1) + g = self.strict_dict_class(("a", "b", "c", "d"))(a=1, b=1, c=1, d=1) + h = self.strict_dict_class(("a", "c", "b"))(a=1, b=1, c=1) + i = self.strict_dict_class(("a", "c", "b"))(a=1, b=1, c=2) + + self.assertEqual(d, dd) + self.assertNotEqual(d, e) + self.assertNotEqual(d, f) + self.assertNotEqual(d, g) + self.assertNotEqual(f, d) + self.assertEqual(d, h) + self.assertNotEqual(d, i) + + def test_setattr_getattr(self): + d = self.dtype() + d.a = 1 + self.assertEqual(d.a, 1) + self.assertRaises(AttributeError, lambda: d.b) + + def test_setattr_raises_on_nonexisting_attr(self): + d = self.dtype() + def _f(): + d.x=1 + self.assertRaises(AttributeError, _f) + + def test_setattr_getattr_special(self): + d = self.strict_dict_class(["items"]) + d.items = 1 + self.assertEqual(d.items, 1) + + def test_get(self): + d = self.dtype(a=1) + self.assertEqual(d.get('a'), 1) + self.assertEqual(d.get('b', 'bla'), 'bla') + + def test_items(self): + d = self.dtype(a=1) + self.assertEqual(d.items(), [('a', 1)]) + d = self.dtype(a=1, b=2) + self.assertEqual(d.items(), [('a', 1), ('b', 2)]) + + def test_mappings_protocol(self): + d = self.dtype(a=1, b=2) + assert dict(d) == {'a': 1, 'b': 2} + assert dict(**d) == {'a': 1, 'b': 2} + + +class TestSemiSrictDict(TestStrictDict): + def strict_dict_class(self, *args, **kwargs): + return SemiStrictDict.create(*args, **kwargs) + + def test_init_fails_on_nonexisting_attrs(self): + # disable irrelevant test + pass + + def test_setattr_raises_on_nonexisting_attr(self): + # disable irrelevant test + pass + + def test_setattr_getattr_nonexisting_attr_succeeds(self): + d = self.dtype() + d.x = 1 + self.assertEqual(d.x, 1) + + def test_init_succeeds_with_nonexisting_attrs(self): + d = self.dtype(a=1, b=1, c=1, x=2) + self.assertEqual((d.a, d.b, d.c, d.x), (1, 1, 1, 2)) + + def test_iter_with_nonexisting_attrs(self): + d = self.dtype(a=1, b=1, c=1, x=2) + self.assertEqual(list(d), ['a', 'b', 'c', 'x']) + + def test_iteritems_with_nonexisting_attrs(self): + d = self.dtype(a=1, b=1, c=1, x=2) + self.assertEqual(list(d.iteritems()), [('a', 1), ('b', 1), ('c', 1), ('x', 2)]) + + def tets_cmp_with_strict_dicts(self): + d = self.dtype(a=1, b=1, c=1) + dd = StrictDict.create(("a", "b", "c"))(a=1, b=1, c=1) + self.assertEqual(d, dd) + + def test_cmp_with_strict_dict_with_nonexisting_attrs(self): + d = self.dtype(a=1, b=1, c=1, x=2) + dd = StrictDict.create(("a", "b", "c", "x"))(a=1, b=1, c=1, x=2) + self.assertEqual(d, dd) + +if __name__ == '__main__': + unittest.main() From 7bb2fe128a4b8c8b08f4d800f15229e2814bfac8 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Thu, 12 Jun 2014 11:08:41 -0300 Subject: [PATCH 44/89] Added PR #657 --- mongoengine/queryset/base.py | 11 +++++------ tests/queryset/queryset.py | 9 ++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 94a6e4b5..cb48f6ca 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -154,22 +154,21 @@ class BaseQuerySet(object): def __iter__(self): raise NotImplementedError - def has_data(self): + def _has_data(self): """ Retrieves whether cursor has any data. """ - queryset = self.clone() - queryset._ordering = [] + queryset = self.order_by() return False if queryset.first() is None else True def __nonzero__(self): """ Avoid to open all records in an if stmt in Py2. """ - return self.has_data() + return self._has_data() def __bool__(self): """ Avoid to open all records in an if stmt in Py3. """ - return self.has_data() + return self._has_data() # Core functions @@ -1410,7 +1409,7 @@ class BaseQuerySet(object): pass key_list.append((key, direction)) - if self._cursor_obj: + if self._cursor_obj and key_list: self._cursor_obj.sort(key_list) return key_list diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 38499df4..a2438e21 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3908,9 +3908,6 @@ class QuerySetTest(unittest.TestCase): raise AssertionError('Cursor has data and it must returns True,' ' even in the last item.') - self.assertTrue(queryset.has_data(), 'Cursor has data and ' - 'returned False') - def test_bool_performance(self): class Person(Document): @@ -3985,10 +3982,12 @@ class QuerySetTest(unittest.TestCase): op = q.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] - self.assertTrue('$orderby' in op['query'], - 'BaseQuerySet cannot remove orderby from meta in boolen test') + self.assertFalse('$orderby' in op['query'], + 'BaseQuerySet must remove orderby from meta in boolen test') self.assertEqual(Person.objects.first().name, 'A') + self.assertTrue(Person.objects._has_data(), + 'Cursor has data and returned False') if __name__ == '__main__': From 03559a3cc4b0ee3a5509c8eb2f1f54bf342284b4 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 24 Jun 2014 19:20:15 +0300 Subject: [PATCH 45/89] Added Python 3.4 to the build process. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5739909b..7ba22694 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "2.7" - "3.2" - "3.3" + - "3.4" env: - PYMONGO=dev DJANGO=1.6 - PYMONGO=dev DJANGO=1.5.5 From bb461b009f6aa0c64c6fb2cf845a4714ee324908 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 24 Jun 2014 19:39:27 +0300 Subject: [PATCH 46/89] Travis build improvements. The latest patch version of each Django minor version is used. The build now installs existing pymongo versions. The build now actually tests against the specified Django version. Replaced PIL with Pillow. Added PyPy and Python 3.4 to the build. Rebase Log: Installing Pillow instead of PIL for testing since it's recommended and it supports PyPy. Excluding Django versions that do not work with Python 3. Improved formatting of .travis.yml. Specifying Pillow 2.0.0 and above since it's the first version that is supported in Python 3. PIL should not be installed alongside Pillow. Also, I installed some libraries that both PIL and Pillow depend on. It seems I have to be explicit on all envvars in order to exclude Django 1.4 from the build matrix. The build is now installing pymongo versions that actually exist. openjpeg has a different name on Ubuntu 12.04. Restoring libz hack. Also installing all Pillow requirements just in case. Fixed the build matrix. Acting according to @BanzaiMan's advice in travis-ci/travis-ci/#1492. --- .travis.yml | 58 +++++++++++++++++++++++++++++++++++++++++------------ setup.py | 4 ++-- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5739909b..d00dde3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,22 +6,54 @@ python: - "2.7" - "3.2" - "3.3" + - "3.4" + - "pypy" env: - - PYMONGO=dev DJANGO=1.6 - - PYMONGO=dev DJANGO=1.5.5 - - PYMONGO=dev DJANGO=1.4.10 - - PYMONGO=2.5 DJANGO=1.6 - - PYMONGO=2.5 DJANGO=1.5.5 - - PYMONGO=2.5 DJANGO=1.4.10 - - PYMONGO=3.2 DJANGO=1.6 - - PYMONGO=3.2 DJANGO=1.5.5 - - PYMONGO=3.3 DJANGO=1.6 - - PYMONGO=3.3 DJANGO=1.5.5 + - PYMONGO=dev DJANGO=1.6.5 + - PYMONGO=dev DJANGO=1.5.8 + - PYMONGO=dev DJANGO=1.4.13 + - PYMONGO=2.5.2 DJANGO=1.6.5 + - PYMONGO=2.5.2 DJANGO=1.5.8 + - PYMONGO=2.5.2 DJANGO=1.4.13 + - PYMONGO=2.6.3 DJANGO=1.6.5 + - PYMONGO=2.6.3 DJANGO=1.5.8 + - PYMONGO=2.6.3 DJANGO=1.4.13 + - PYMONGO=2.7.1 DJANGO=1.6.5 + - PYMONGO=2.7.1 DJANGO=1.5.8 + - PYMONGO=2.7.1 DJANGO=1.4.13 + +matrix: + exclude: + - python: "3.2" + env: PYMONGO=dev DJANGO=1.4.13 + - python: "3.2" + env: PYMONGO=2.5.2 DJANGO=1.4.13 + - python: "3.2" + env: PYMONGO=2.6.3 DJANGO=1.4.13 + - python: "3.2" + env: PYMONGO=2.7.1 DJANGO=1.4.13 + - python: "3.3" + env: PYMONGO=dev DJANGO=1.4.13 + - python: "3.3" + env: PYMONGO=2.5.2 DJANGO=1.4.13 + - python: "3.3" + env: PYMONGO=2.6.3 DJANGO=1.4.13 + - python: "3.3" + env: PYMONGO=2.7.1 DJANGO=1.4.13 + - python: "3.4" + env: PYMONGO=dev DJANGO=1.4.13 + - python: "3.4" + env: PYMONGO=2.5.2 DJANGO=1.4.13 + - python: "3.4" + env: PYMONGO=2.6.3 DJANGO=1.4.13 + - python: "3.4" + env: PYMONGO=2.7.1 DJANGO=1.4.13 install: - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi + - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk + - cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/ - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi + - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO; true; fi + - pip install Django==$DJANGO - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b - python setup.py install script: diff --git a/setup.py b/setup.py index 85707d00..40c27ebe 100644 --- a/setup.py +++ b/setup.py @@ -51,12 +51,12 @@ CLASSIFIERS = [ extra_opts = {"packages": find_packages(exclude=["tests", "tests.*"])} if sys.version_info[0] == 3: extra_opts['use_2to3'] = True - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'Pillow>=2.0.0', 'django>=1.5.1'] if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'] = find_packages() extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'Pillow>=2.0.0', 'jinja2>=2.6', 'python-dateutil'] setup(name='mongoengine', version=VERSION, From 8e852bce02b14a0149c0e189642631edc2de31fe Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 10:58:00 +0300 Subject: [PATCH 47/89] Pillow provides a more descriptive error message, therefor the build failure. --- tests/fields/file_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index 902b1512..7ae53e8a 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -279,7 +279,7 @@ class FileTest(unittest.TestCase): t.image.put(f) self.fail("Should have raised an invalidation error") except ValidationError, e: - self.assertEqual("%s" % e, "Invalid image: cannot identify image file") + self.assertEqual("%s" % e, "Invalid image: cannot identify image file %s" % f) t = TestImage() t.image.put(open(TEST_IMAGE_PATH, 'rb')) From adbbc656d41dbc7ecd33ae933c164efe3fd89316 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 11:12:40 +0300 Subject: [PATCH 48/89] Removing zlib hack since only PIL needs it. The build should pass without it. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d00dde3a..1265ba70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,6 @@ matrix: env: PYMONGO=2.7.1 DJANGO=1.4.13 install: - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk - - cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/ - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO; true; fi - pip install Django==$DJANGO From 8adf1cdd0212e18d26220d7e5997c3534c92a017 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 11:18:35 +0300 Subject: [PATCH 49/89] Fast finish the build if there are failures since we have a very large build matrix and each build takes a very long time. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1265ba70..4079855d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ env: - PYMONGO=2.7.1 DJANGO=1.4.13 matrix: + fast_finish: true exclude: - python: "3.2" env: PYMONGO=dev DJANGO=1.4.13 From fc3eda55c72e99f0b80bb37bf917099780c2f79e Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 11:32:41 +0300 Subject: [PATCH 50/89] Added a note about optional dependencies to the README file. --- README.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index cc4524ae..8c3ee26e 100644 --- a/README.rst +++ b/README.rst @@ -29,9 +29,18 @@ setup.py install``. Dependencies ============ -- pymongo 2.5+ +- pymongo>=2.5 - sphinx (optional - for documentation generation) +Optional Dependencies +--------------------- +- **Django Integration:** Django>=1.4.0 for Python 2.x or PyPy and Django>=1.5.0 for Python 3.x +- **Image Fields**: Pillow>=2.0.0 or PIL (not recommended since MongoEngine is tested with Pillow) +- dateutil>=2.1.0 + +.. note + MongoEngine always runs it's test suite against the latest patch version of each dependecy. e.g.: Django 1.6.5 + Examples ======== Some simple examples of what MongoEngine code looks like:: From fe2ef4e61c4bd44e5156542728b7755f30ef8148 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 11:39:08 +0300 Subject: [PATCH 51/89] Made the benchmark script compatitable with Python 3 and ensured it runs on every build. --- .travis.yml | 1 + benchmark.py | 41 +++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4079855d..fff35ca9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,7 @@ install: - python setup.py install script: - python setup.py test + - python benchmark.py notifications: irc: "irc.freenode.org#mongoengine" branches: diff --git a/benchmark.py b/benchmark.py index 16b2fd47..cb5e19d8 100644 --- a/benchmark.py +++ b/benchmark.py @@ -113,6 +113,7 @@ def main(): 4.68946313858 ---------------------------------------------------------------------------------------------------- """ + print("Benchmarking...") setup = """ from pymongo import MongoClient @@ -138,10 +139,10 @@ myNoddys = noddy.find() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries - Pymongo""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ from pymongo import MongoClient @@ -161,10 +162,10 @@ myNoddys = noddy.find() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) setup = """ from pymongo import MongoClient @@ -190,10 +191,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine""" + print("-" * 100) + print("""Creating 10000 dictionaries - MongoEngine""") t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -208,10 +209,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries without continual assign - MongoEngine""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -224,10 +225,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -240,10 +241,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -256,10 +257,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""" + print("-" * 100) + print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""") t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -272,11 +273,11 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""" + print("-" * 100) + print("""Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""") t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) if __name__ == "__main__": - main() \ No newline at end of file + main() From f44c8f120532f407b5eea6fed480183c8aa51758 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 13:11:32 +0300 Subject: [PATCH 52/89] Skipping a test that does not work on PyPy due to a PyPy bug/feature. --- tests/queryset/queryset.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7ff2965d..33f7dd91 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3586,7 +3586,13 @@ class QuerySetTest(unittest.TestCase): [x for x in people] self.assertEqual(100, len(people._result_cache)) - self.assertEqual(None, people._len) + + import platform + + if platform.python_implementation() != "PyPy": + # PyPy evaluates __len__ when iterating with list comprehensions while CPython does not. + # This may be a bug in PyPy (PyPy/#1802) but it does not affect the behavior of MongoEngine. + self.assertEqual(None, people._len) self.assertEqual(q, 1) list(people) From 1914032e35ef917ea95efc192b8280858ff78c82 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 14:20:54 +0300 Subject: [PATCH 53/89] Missed some of the print statements in the benchmarks script. --- benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark.py b/benchmark.py index cb5e19d8..5613eaf8 100644 --- a/benchmark.py +++ b/benchmark.py @@ -210,7 +210,7 @@ myNoddys = Noddy.objects() """ print("-" * 100) - print """Creating 10000 dictionaries without continual assign - MongoEngine""" + print("""Creating 10000 dictionaries without continual assign - MongoEngine""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) @@ -242,7 +242,7 @@ myNoddys = Noddy.objects() """ print("-" * 100) - print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""" + print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) From 7f7745071af87957cdc5705b409b96ed49a1e481 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 15:47:54 +0300 Subject: [PATCH 54/89] Found more print statements that were not turned into function calls. --- benchmark.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark.py b/benchmark.py index 5613eaf8..d33e58db 100644 --- a/benchmark.py +++ b/benchmark.py @@ -140,7 +140,7 @@ myNoddys = noddy.find() """ print("-" * 100) - print """Creating 10000 dictionaries - Pymongo""" + print("""Creating 10000 dictionaries - Pymongo""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) @@ -163,7 +163,7 @@ myNoddys = noddy.find() """ print("-" * 100) - print """Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""" + print("""Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) @@ -226,7 +226,7 @@ myNoddys = Noddy.objects() """ print("-" * 100) - print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""" + print("""Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) From 29309dac9a926077962fc0778e5b2fbaf1d29cc2 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 16:53:24 +0300 Subject: [PATCH 55/89] Mongo clients with the same settings should be shared since they manage a connection pool. Also, I removed old code that was supposed to support Pymongo<2.1 which we don't support anymore. --- mongoengine/connection.py | 33 ++++++++++++++++++--------------- tests/test_connection.py | 11 +++++++++++ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 7cc626f4..d3efac62 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -93,20 +93,11 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): raise ConnectionError(msg) conn_settings = _connection_settings[alias].copy() - if hasattr(pymongo, 'version_tuple'): # Support for 2.1+ - conn_settings.pop('name', None) - conn_settings.pop('slaves', None) - conn_settings.pop('is_slave', None) - conn_settings.pop('username', None) - conn_settings.pop('password', None) - else: - # Get all the slave connections - if 'slaves' in conn_settings: - slaves = [] - for slave_alias in conn_settings['slaves']: - slaves.append(get_connection(slave_alias)) - conn_settings['slaves'] = slaves - conn_settings.pop('read_preference', None) + conn_settings.pop('name', None) + conn_settings.pop('slaves', None) + conn_settings.pop('is_slave', None) + conn_settings.pop('username', None) + conn_settings.pop('password', None) connection_class = MongoClient if 'replicaSet' in conn_settings: @@ -119,7 +110,19 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): connection_class = MongoReplicaSetClient try: - _connections[alias] = connection_class(**conn_settings) + connection = None + connection_settings_iterator = ((alias, settings.copy()) for alias, settings in _connection_settings.iteritems()) + for alias, connection_settings in connection_settings_iterator: + connection_settings.pop('name', None) + connection_settings.pop('slaves', None) + connection_settings.pop('is_slave', None) + connection_settings.pop('username', None) + connection_settings.pop('password', None) + if conn_settings == connection_settings and _connections.get(alias, None): + connection = _connections[alias] + break + + _connections[alias] = connection if connection else connection_class(**conn_settings) except Exception, e: raise ConnectionError("Cannot connect to database %s :\n%s" % (alias, e)) return _connections[alias] diff --git a/tests/test_connection.py b/tests/test_connection.py index 96135bc5..bf615ceb 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -34,6 +34,17 @@ class ConnectionTest(unittest.TestCase): conn = get_connection('testdb') self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) + def test_sharing_connections(self): + """Ensure that connections are shared when the connection settings are exactly the same + """ + connect('mongoenginetest', alias='testdb1') + + expected_connection = get_connection('testdb1') + + connect('mongoenginetest', alias='testdb2') + actual_connection = get_connection('testdb2') + self.assertIs(expected_connection, actual_connection) + def test_connect_uri(self): """Ensure that the connect() method works properly with uri's """ From b8d568761e0e3130ce37d5675f8b218b11b88e29 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 17:24:52 +0300 Subject: [PATCH 56/89] Getting rid of xrange since it's not in Python 3 and does not affect the benchmark. --- benchmark.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmark.py b/benchmark.py index d33e58db..53ecf32c 100644 --- a/benchmark.py +++ b/benchmark.py @@ -15,7 +15,7 @@ def cprofile_main(): class Noddy(Document): fields = DictField() - for i in xrange(1): + for i in range(1): noddy = Noddy() for j in range(20): noddy.fields["key" + str(j)] = "value " + str(j) @@ -128,7 +128,7 @@ connection = MongoClient() db = connection.timeit_test noddy = db.noddy -for i in xrange(10000): +for i in range(10000): example = {'fields': {}} for j in range(20): example['fields']["key"+str(j)] = "value "+str(j) @@ -151,7 +151,7 @@ connection = MongoClient() db = connection.timeit_test noddy = db.noddy -for i in xrange(10000): +for i in range(10000): example = {'fields': {}} for j in range(20): example['fields']["key"+str(j)] = "value "+str(j) @@ -181,7 +181,7 @@ class Noddy(Document): """ stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) @@ -197,7 +197,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() fields = {} for j in range(20): @@ -215,7 +215,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) @@ -231,7 +231,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) @@ -247,7 +247,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) @@ -263,7 +263,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) From 5ae588833b7ba1498a95e2efda4f788d4297997b Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 18:22:39 +0300 Subject: [PATCH 57/89] Allowed to switch databases for a specific query. --- mongoengine/context_managers.py | 7 ------- mongoengine/queryset/base.py | 24 ++++++++++++++++++++---- mongoengine/queryset/queryset.py | 7 +++++++ tests/queryset/queryset.py | 16 ++++++++++++++++ 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 13ed1009..cc860066 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -1,6 +1,5 @@ from mongoengine.common import _import_class from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db -from mongoengine.queryset import QuerySet __all__ = ("switch_db", "switch_collection", "no_dereference", @@ -162,12 +161,6 @@ class no_sub_classes(object): return self.cls -class QuerySetNoDeRef(QuerySet): - """Special no_dereference QuerySet""" - def __dereference(items, max_depth=1, instance=None, name=None): - return items - - class query_counter(object): """ Query_counter context manager to get the number of queries. """ diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c2ad027e..a8d204b3 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -13,11 +13,11 @@ import pymongo from pymongo.common import validate_read_preference from mongoengine import signals +from mongoengine.context_managers import switch_db from mongoengine.common import _import_class from mongoengine.base.common import get_document from mongoengine.errors import (OperationError, NotUniqueError, InvalidQueryError, LookUpError) - from mongoengine.queryset import transform from mongoengine.queryset.field_list import QueryFieldList from mongoengine.queryset.visitor import Q, QNode @@ -389,7 +389,7 @@ class BaseQuerySet(object): ref_q = document_cls.objects(**{field_name + '__in': self}) ref_q_count = ref_q.count() if (doc != document_cls and ref_q_count > 0 - or (doc == document_cls and ref_q_count > 0)): + or (doc == document_cls and ref_q_count > 0)): ref_q.delete(write_concern=write_concern) elif rule == NULLIFY: document_cls.objects(**{field_name + '__in': self}).update( @@ -522,6 +522,19 @@ class BaseQuerySet(object): return self + def using(self, alias): + """This method is for controlling which database the QuerySet will be evaluated against if you are using more than one database. + + :param alias: The database alias + + .. versionadded:: 0.8 + """ + + with switch_db(self._document, alias) as cls: + collection = cls._get_collection() + + return self.clone_into(self.__class__(self._document, collection)) + def clone(self): """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet` @@ -926,7 +939,7 @@ class BaseQuerySet(object): mr_args['out'] = output results = getattr(queryset._collection, map_reduce_function)( - map_f, reduce_f, **mr_args) + map_f, reduce_f, **mr_args) if map_reduce_function == 'map_reduce': results = results.find() @@ -1362,7 +1375,7 @@ class BaseQuerySet(object): for subdoc in subclasses: try: subfield = ".".join(f.db_field for f in - subdoc._lookup_field(field.split('.'))) + subdoc._lookup_field(field.split('.'))) ret.append(subfield) found = True break @@ -1450,6 +1463,7 @@ class BaseQuerySet(object): # type of this field and use the corresponding # .to_python(...) from mongoengine.fields import EmbeddedDocumentField + obj = self._document for chunk in path.split('.'): obj = getattr(obj, chunk, None) @@ -1460,6 +1474,7 @@ class BaseQuerySet(object): if obj and data is not None: data = obj.to_python(data) return data + return clean(row) def _sub_js_fields(self, code): @@ -1468,6 +1483,7 @@ class BaseQuerySet(object): substituted for the MongoDB name of the field (specified using the :attr:`name` keyword argument in a field's constructor). """ + def field_sub(match): # Extract just the field name, and look up the field objects field_name = match.group(1).split('.') diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 1437e76b..cebfcc50 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -155,3 +155,10 @@ class QuerySetNoCache(BaseQuerySet): queryset = self.clone() queryset.rewind() return queryset + + +class QuerySetNoDeRef(QuerySet): + """Special no_dereference QuerySet""" + + def __dereference(items, max_depth=1, instance=None, name=None): + return items \ No newline at end of file diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7ff2965d..25ede817 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -29,6 +29,7 @@ class QuerySetTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') + connect(db='mongoenginetest2', alias='test2') class PersonMeta(EmbeddedDocument): weight = IntField() @@ -2957,6 +2958,21 @@ class QuerySetTest(unittest.TestCase): Number.drop_collection() + def test_using(self): + """Ensure that switching databases for a queryset is possible + """ + class Number2(Document): + n = IntField() + + Number2.drop_collection() + + for i in xrange(1, 10): + t = Number2(n=i) + t.switch_db('test2') + t.save() + + self.assertEqual(len(Number2.objects.using('test2')), 9) + def test_unset_reference(self): class Comment(Document): text = StringField() From 67a65a2aa9ecd77de372511aa54541100923d3d3 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 26 Jun 2014 11:17:57 +0300 Subject: [PATCH 58/89] Installing unittest2 on Python 2.6. --- setup.py | 3 +++ tests/test_connection.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 85707d00..f8f0d6da 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,9 @@ if sys.version_info[0] == 3: else: extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil'] + if sys.version_info[0] == 2 and sys.version_info[1] == 6: + extra_opts['tests_require'].append('unittest2') + setup(name='mongoengine', version=VERSION, author='Harry Marr', diff --git a/tests/test_connection.py b/tests/test_connection.py index bf615ceb..a5b1b089 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,6 +1,11 @@ import sys sys.path[0:0] = [""] -import unittest + +try: + import unittest2 as unittest +except ImportError: + import unittest + import datetime import pymongo From cae91ce0c564089b418d0fd508d80c7f923df949 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 26 Jun 2014 12:31:07 +0300 Subject: [PATCH 59/89] Convert codebase to Python 3 using 2to3 before running benchmarks. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fff35ca9..6281e6b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,7 @@ install: - python setup.py install script: - python setup.py test + - 2to3 . -w - python benchmark.py notifications: irc: "irc.freenode.org#mongoengine" From 0e7878b406bc23b46140edd54912e4491595fac4 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 26 Jun 2014 12:41:26 +0300 Subject: [PATCH 60/89] Only run 2to3 on Python 3.x. Makes sense no? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6281e6b4..5f10339b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,7 +58,7 @@ install: - python setup.py install script: - python setup.py test - - 2to3 . -w + - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then 2to3 . -w; fi; - python benchmark.py notifications: irc: "irc.freenode.org#mongoengine" From b02a31d4b920981f6b7330de23e78518f334cc0d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 14:44:44 +0100 Subject: [PATCH 61/89] Updated .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5739909b..c7397fd3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,7 @@ python: - "3.3" env: - PYMONGO=dev DJANGO=1.6 - - PYMONGO=dev DJANGO=1.5.5 - - PYMONGO=dev DJANGO=1.4.10 + - PYMONGO=dev DJANGO=1.5.8 - PYMONGO=2.5 DJANGO=1.6 - PYMONGO=2.5 DJANGO=1.5.5 - PYMONGO=2.5 DJANGO=1.4.10 @@ -31,3 +30,4 @@ notifications: branches: only: - master + - "0.8" From dd51589f67fd02f3049aadea5268a79cd3799092 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 16:02:40 +0100 Subject: [PATCH 62/89] Updates --- setup.py | 2 +- tests/document/instance.py | 4 ++-- tests/queryset/queryset.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 85707d00..25075275 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setup(name='mongoengine', long_description=LONG_DESCRIPTION, platforms=['any'], classifiers=CLASSIFIERS, - install_requires=['pymongo>=2.5'], + install_requires=['pymongo>=2.7'], test_suite='nose.collector', **extra_opts ) diff --git a/tests/document/instance.py b/tests/document/instance.py index afb27e0d..6f04ac1d 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -57,7 +57,7 @@ class InstanceTest(unittest.TestCase): date = DateTimeField(default=datetime.now) meta = { 'max_documents': 10, - 'max_size': 90000, + 'max_size': 4096, } Log.drop_collection() @@ -75,7 +75,7 @@ class InstanceTest(unittest.TestCase): options = Log.objects._collection.options() self.assertEqual(options['capped'], True) self.assertEqual(options['max'], 10) - self.assertEqual(options['size'], 90000) + self.assertTrue(options['size'] >= 4096) # Check that the document cannot be redefined with different options def recreate_log_document(): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index a2438e21..688b1bce 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -650,7 +650,7 @@ class QuerySetTest(unittest.TestCase): blogs.append(Blog(title="post %s" % i, posts=[post1, post2])) Blog.objects.insert(blogs, load_bulk=False) - self.assertEqual(q, 1) # 1 for the insert + self.assertEqual(q, 99) # profiling logs each doc now :( Blog.drop_collection() Blog.ensure_indexes() @@ -659,7 +659,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(q, 0) Blog.objects.insert(blogs) - self.assertEqual(q, 2) # 1 for insert, and 1 for in bulk fetch + self.assertEqual(q, 100) # 99 or insert, and 1 for in bulk fetch Blog.drop_collection() @@ -3943,7 +3943,7 @@ class QuerySetTest(unittest.TestCase): if qs: pass - op = q.db.system.profile.find({"ns": + op = q.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] self.assertFalse('$orderby' in op['query'], @@ -3969,7 +3969,7 @@ class QuerySetTest(unittest.TestCase): } Person.drop_collection() - + Person(name="B").save() Person(name="C").save() Person(name="A").save() @@ -3979,13 +3979,13 @@ class QuerySetTest(unittest.TestCase): if Person.objects: pass - op = q.db.system.profile.find({"ns": + op = q.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] self.assertFalse('$orderby' in op['query'], 'BaseQuerySet must remove orderby from meta in boolen test') - self.assertEqual(Person.objects.first().name, 'A') + self.assertEqual(Person.objects.first().name, 'A') self.assertTrue(Person.objects._has_data(), 'Cursor has data and returned False') From 5b6c8c191f1418471d8b9f616d5fbded4a797238 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 14:44:44 +0100 Subject: [PATCH 63/89] Updated .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5739909b..3d9d6611 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,7 @@ python: - "3.3" env: - PYMONGO=dev DJANGO=1.6 - - PYMONGO=dev DJANGO=1.5.5 - - PYMONGO=dev DJANGO=1.4.10 + - PYMONGO=dev DJANGO=1.5.8 - PYMONGO=2.5 DJANGO=1.6 - PYMONGO=2.5 DJANGO=1.5.5 - PYMONGO=2.5 DJANGO=1.4.10 @@ -31,3 +30,4 @@ notifications: branches: only: - master + - "0.9" From eb9003187d4ce70bdbb32709fc6cb9f20ce7eb16 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 16:13:01 +0100 Subject: [PATCH 64/89] Updated changelog & authors #673 --- AUTHORS | 1 + docs/changelog.rst | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 170a00e5..c86df67c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -191,3 +191,4 @@ that much better: * Damien Churchill (https://github.com/damoxc) * Jonathan Simon Prates (https://github.com/jonathansp) * Thiago Papageorgiou (https://github.com/tmpapageorgiou) + * Omer Katz (https://github.com/thedrow) diff --git a/docs/changelog.rst b/docs/changelog.rst index 51134238..b94388a1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= + +Changes in 0.9.X - DEV +====================== +- pypy support #673 + Changes in 0.8.7 ================ - Calling reload on deleted / nonexistant documents raises DoesNotExist (#538) From 11724aa5555ad7f87e0208b3ce01ac8b7f31323c Mon Sep 17 00:00:00 2001 From: Dmitry Konishchev Date: Thu, 26 Jun 2014 16:18:42 +0100 Subject: [PATCH 65/89] QuerySet.modify() method to provide find_and_modify() like behaviour --- docs/guide/querying.rst | 5 +- mongoengine/queryset/base.py | 54 +++++++++++++++++++ tests/queryset/__init__.py | 1 + tests/queryset/modify.py | 102 +++++++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 tests/queryset/modify.py diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 32cbb94e..96beea5f 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -488,8 +488,9 @@ calling it with keyword arguments:: Atomic updates ============== Documents may be updated atomically by using the -:meth:`~mongoengine.queryset.QuerySet.update_one` and -:meth:`~mongoengine.queryset.QuerySet.update` methods on a +:meth:`~mongoengine.queryset.QuerySet.update_one`, +:meth:`~mongoengine.queryset.QuerySet.update` and +:meth:`~mongoengine.queryset.QuerySet.modify` methods on a :meth:`~mongoengine.queryset.QuerySet`. There are several different "modifiers" that you may use with these methods: diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 89a5e5fb..db60deba 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -10,6 +10,7 @@ import warnings from bson.code import Code from bson import json_util import pymongo +import pymongo.errors from pymongo.common import validate_read_preference from mongoengine import signals @@ -484,6 +485,59 @@ class BaseQuerySet(object): return self.update( upsert=upsert, multi=False, write_concern=write_concern, **update) + def modify(self, upsert=False, full_response=False, remove=False, new=False, **update): + """Update and return the updated document. + + Returns either the document before or after modification based on `new` + parameter. If no documents match the query and `upsert` is false, + returns ``None``. If upserting and `new` is false, returns ``None``. + + If the full_response parameter is ``True``, the return value will be + the entire response object from the server, including the 'ok' and + 'lastErrorObject' fields, rather than just the modified document. + This is useful mainly because the 'lastErrorObject' document holds + information about the command's execution. + + :param upsert: insert if document doesn't exist (default ``False``) + :param full_response: return the entire response object from the + server (default ``False``) + :param remove: remove rather than updating (default ``False``) + :param new: return updated rather than original document + (default ``False``) + :param update: Django-style update keyword arguments + + .. versionadded:: 0.9 + """ + + if remove and new: + raise OperationError("Conflicting parameters: remove and new") + + if not update and not upsert and not remove: + raise OperationError("No update parameters, must either update or remove") + + queryset = self.clone() + query = queryset._query + update = transform.update(queryset._document, **update) + sort = queryset._ordering + + try: + result = queryset._collection.find_and_modify( + query, update, upsert=upsert, sort=sort, remove=remove, new=new, + full_response=full_response, **self._cursor_args) + except pymongo.errors.DuplicateKeyError, err: + raise NotUniqueError(u"Update failed (%s)" % err) + except pymongo.errors.OperationFailure, err: + raise OperationError(u"Update failed (%s)" % err) + + if full_response: + if result["value"] is not None: + result["value"] = self._document._from_son(result["value"]) + else: + if result is not None: + result = self._document._from_son(result) + + return result + def with_id(self, object_id): """Retrieve the object matching the id provided. Uses `object_id` only and raises InvalidQueryError if a filter has been applied. Returns diff --git a/tests/queryset/__init__.py b/tests/queryset/__init__.py index 8a93c19f..c36b2684 100644 --- a/tests/queryset/__init__.py +++ b/tests/queryset/__init__.py @@ -3,3 +3,4 @@ from field_list import * from queryset import * from visitor import * from geo import * +from modify import * \ No newline at end of file diff --git a/tests/queryset/modify.py b/tests/queryset/modify.py new file mode 100644 index 00000000..e0c7d1fe --- /dev/null +++ b/tests/queryset/modify.py @@ -0,0 +1,102 @@ +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import connect, Document, IntField + +__all__ = ("FindAndModifyTest",) + + +class Doc(Document): + id = IntField(primary_key=True) + value = IntField() + + +class FindAndModifyTest(unittest.TestCase): + + def setUp(self): + connect(db="mongoenginetest") + Doc.drop_collection() + + def assertDbEqual(self, docs): + self.assertEqual(list(Doc._collection.find().sort("id")), docs) + + def test_modify(self): + Doc(id=0, value=0).save() + doc = Doc(id=1, value=1).save() + + old_doc = Doc.objects(id=1).modify(set__value=-1) + self.assertEqual(old_doc.to_json(), doc.to_json()) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}]) + + def test_modify_with_new(self): + Doc(id=0, value=0).save() + doc = Doc(id=1, value=1).save() + + new_doc = Doc.objects(id=1).modify(set__value=-1, new=True) + doc.value = -1 + self.assertEqual(new_doc.to_json(), doc.to_json()) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}]) + + def test_modify_not_existing(self): + Doc(id=0, value=0).save() + self.assertEqual(Doc.objects(id=1).modify(set__value=-1), None) + self.assertDbEqual([{"_id": 0, "value": 0}]) + + def test_modify_with_upsert(self): + Doc(id=0, value=0).save() + old_doc = Doc.objects(id=1).modify(set__value=1, upsert=True) + self.assertEqual(old_doc, None) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}]) + + def test_modify_with_upsert_existing(self): + Doc(id=0, value=0).save() + doc = Doc(id=1, value=1).save() + + old_doc = Doc.objects(id=1).modify(set__value=-1, upsert=True) + self.assertEqual(old_doc.to_json(), doc.to_json()) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}]) + + def test_modify_with_upsert_with_new(self): + Doc(id=0, value=0).save() + new_doc = Doc.objects(id=1).modify(upsert=True, new=True, set__value=1) + self.assertEqual(new_doc.to_mongo(), {"_id": 1, "value": 1}) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}]) + + def test_modify_with_remove(self): + Doc(id=0, value=0).save() + doc = Doc(id=1, value=1).save() + + old_doc = Doc.objects(id=1).modify(remove=True) + self.assertEqual(old_doc.to_json(), doc.to_json()) + self.assertDbEqual([{"_id": 0, "value": 0}]) + + def test_find_and_modify_with_remove_not_existing(self): + Doc(id=0, value=0).save() + self.assertEqual(Doc.objects(id=1).modify(remove=True), None) + self.assertDbEqual([{"_id": 0, "value": 0}]) + + def test_modify_with_order_by(self): + Doc(id=0, value=3).save() + Doc(id=1, value=2).save() + Doc(id=2, value=1).save() + doc = Doc(id=3, value=0).save() + + old_doc = Doc.objects().order_by("-id").modify(set__value=-1) + self.assertEqual(old_doc.to_json(), doc.to_json()) + self.assertDbEqual([ + {"_id": 0, "value": 3}, {"_id": 1, "value": 2}, + {"_id": 2, "value": 1}, {"_id": 3, "value": -1}]) + + def test_modify_with_fields(self): + Doc(id=0, value=0).save() + Doc(id=1, value=1).save() + + old_doc = Doc.objects(id=1).only("id").modify(set__value=-1) + self.assertEqual(old_doc.to_mongo(), {"_id": 1}) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}]) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 2c07d7736847fa1157ab38c6247ecaa0ac34c0e1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 16:24:37 +0100 Subject: [PATCH 66/89] Updated changelog Enabled connection pooling --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b94388a1..bead491c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== - pypy support #673 +- Enabled connection pooling #674 Changes in 0.8.7 ================ From d1d59722774f59d3ee9b935bfda9ff7e373b43c3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 16:34:02 +0100 Subject: [PATCH 67/89] Removed support for old versions - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x. - Removing support for Python < 2.6.6 --- .travis.yml | 34 +-------------------------------- docs/changelog.rst | 2 ++ mongoengine/base/document.py | 7 +------ mongoengine/python_support.py | 32 ------------------------------- mongoengine/queryset/visitor.py | 5 +++-- setup.py | 6 ++++-- 6 files changed, 11 insertions(+), 75 deletions(-) diff --git a/.travis.yml b/.travis.yml index c20e52af..40736165 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,44 +11,12 @@ python: env: - PYMONGO=dev DJANGO=1.6.5 - PYMONGO=dev DJANGO=1.5.8 - - PYMONGO=dev DJANGO=1.4.13 - - PYMONGO=2.5.2 DJANGO=1.6.5 - - PYMONGO=2.5.2 DJANGO=1.5.8 - - PYMONGO=2.5.2 DJANGO=1.4.13 - - PYMONGO=2.6.3 DJANGO=1.6.5 - - PYMONGO=2.6.3 DJANGO=1.5.8 - - PYMONGO=2.6.3 DJANGO=1.4.13 - PYMONGO=2.7.1 DJANGO=1.6.5 - PYMONGO=2.7.1 DJANGO=1.5.8 - - PYMONGO=2.7.1 DJANGO=1.4.13 matrix: fast_finish: true - exclude: - - python: "3.2" - env: PYMONGO=dev DJANGO=1.4.13 - - python: "3.2" - env: PYMONGO=2.5.2 DJANGO=1.4.13 - - python: "3.2" - env: PYMONGO=2.6.3 DJANGO=1.4.13 - - python: "3.2" - env: PYMONGO=2.7.1 DJANGO=1.4.13 - - python: "3.3" - env: PYMONGO=dev DJANGO=1.4.13 - - python: "3.3" - env: PYMONGO=2.5.2 DJANGO=1.4.13 - - python: "3.3" - env: PYMONGO=2.6.3 DJANGO=1.4.13 - - python: "3.3" - env: PYMONGO=2.7.1 DJANGO=1.4.13 - - python: "3.4" - env: PYMONGO=dev DJANGO=1.4.13 - - python: "3.4" - env: PYMONGO=2.5.2 DJANGO=1.4.13 - - python: "3.4" - env: PYMONGO=2.6.3 DJANGO=1.4.13 - - python: "3.4" - env: PYMONGO=2.7.1 DJANGO=1.4.13 + install: - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi diff --git a/docs/changelog.rst b/docs/changelog.rst index bead491c..c980e904 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,8 @@ Changes in 0.9.X - DEV ====================== - pypy support #673 - Enabled connection pooling #674 +- Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x. +- Removing support for Python < 2.6.6 Changes in 0.8.7 ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index f5eae8ff..43b865ce 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -13,8 +13,7 @@ from mongoengine import signals from mongoengine.common import _import_class from mongoengine.errors import (ValidationError, InvalidDocumentError, LookUpError) -from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, - to_str_keys_recursive) +from mongoengine.python_support import PY3, txt_type from mongoengine.base.common import get_document, ALLOW_INHERITANCE from mongoengine.base.datastructures import BaseDict, BaseList @@ -545,10 +544,6 @@ class BaseDocument(object): # class if unavailable class_name = son.get('_cls', cls._class_name) data = dict(("%s" % key, value) for key, value in son.iteritems()) - if not UNICODE_KWARGS: - # python 2.6.4 and lower cannot handle unicode keys - # passed to class constructor example: cls(**data) - to_str_keys_recursive(data) # Return correct subclass for document type if class_name != cls._class_name: diff --git a/mongoengine/python_support.py b/mongoengine/python_support.py index 097740eb..2c4df00c 100644 --- a/mongoengine/python_support.py +++ b/mongoengine/python_support.py @@ -3,8 +3,6 @@ import sys PY3 = sys.version_info[0] == 3 -PY25 = sys.version_info[:2] == (2, 5) -UNICODE_KWARGS = int(''.join([str(x) for x in sys.version_info[:3]])) > 264 if PY3: import codecs @@ -29,33 +27,3 @@ else: txt_type = unicode str_types = (bin_type, txt_type) - -if PY25: - def product(*args, **kwds): - pools = map(tuple, args) * kwds.get('repeat', 1) - result = [[]] - for pool in pools: - result = [x + [y] for x in result for y in pool] - for prod in result: - yield tuple(prod) - reduce = reduce -else: - from itertools import product - from functools import reduce - - -# For use with Python 2.5 -# converts all keys from unicode to str for d and all nested dictionaries -def to_str_keys_recursive(d): - if isinstance(d, list): - for val in d: - if isinstance(val, (dict, list)): - to_str_keys_recursive(val) - elif isinstance(d, dict): - for key, val in d.items(): - if isinstance(val, (dict, list)): - to_str_keys_recursive(val) - if isinstance(key, unicode): - d[str(key)] = d.pop(key) - else: - raise ValueError("non list/dict parameter not allowed") diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 41f4ebf8..a39b05f0 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -1,8 +1,9 @@ import copy -from mongoengine.errors import InvalidQueryError -from mongoengine.python_support import product, reduce +from itertools import product +from functools import reduce +from mongoengine.errors import InvalidQueryError from mongoengine.queryset import transform __all__ = ('Q',) diff --git a/setup.py b/setup.py index d37cede1..7270331a 100644 --- a/setup.py +++ b/setup.py @@ -38,12 +38,14 @@ CLASSIFIERS = [ 'Operating System :: OS Independent', 'Programming Language :: Python', "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.6.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.1", "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", 'Topic :: Database', 'Topic :: Software Development :: Libraries :: Python Modules', ] From da0a1bbe9f63059bd745d378e723c184c079c90c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 17:13:21 +0100 Subject: [PATCH 68/89] Fix test_using --- tests/queryset/queryset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 649a3582..b97f9519 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -16,7 +16,7 @@ from bson import ObjectId from mongoengine import * from mongoengine.connection import get_connection from mongoengine.python_support import PY3 -from mongoengine.context_managers import query_counter +from mongoengine.context_managers import query_counter, switch_db from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, queryset_manager) @@ -3035,8 +3035,10 @@ class QuerySetTest(unittest.TestCase): n = IntField() Number2.drop_collection() + with switch_db(Number2, 'test2') as Number2: + Number2.drop_collection() - for i in xrange(1, 10): + for i in range(1, 10): t = Number2(n=i) t.switch_db('test2') t.save() From cfc31eead324e71958c2e69b0117b515a0ae6bc2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 17:13:35 +0100 Subject: [PATCH 69/89] Fixed $maxDistance location for geoJSON $near queries with MongoDB 2.6+ Closes #664 --- mongoengine/queryset/transform.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 27e41ad2..8e88e9fe 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -3,6 +3,7 @@ from collections import defaultdict import pymongo from bson import SON +from mongoengine.connection import get_connection from mongoengine.common import _import_class from mongoengine.errors import InvalidQueryError, LookUpError @@ -115,14 +116,21 @@ def query(_doc_cls=None, _field_operation=False, **query): if key in mongo_query and isinstance(mongo_query[key], dict): mongo_query[key].update(value) # $maxDistance needs to come last - convert to SON - if '$maxDistance' in mongo_query[key]: - value_dict = mongo_query[key] + value_dict = mongo_query[key] + if ('$maxDistance' in value_dict and '$near' in value_dict and + isinstance(value_dict['$near'], dict)): + value_son = SON() for k, v in value_dict.iteritems(): if k == '$maxDistance': continue value_son[k] = v - value_son['$maxDistance'] = value_dict['$maxDistance'] + if (get_connection().max_wire_version <= 1): + value_son['$maxDistance'] = value_dict['$maxDistance'] + else: + value_son['$near'] = SON(value_son['$near']) + value_son['$near']['$maxDistance'] = value_dict['$maxDistance'] + mongo_query[key] = value_son else: # Store for manually merging later From 70651ce99499d354a93c4776548eb48cc748523a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 19:24:52 +0100 Subject: [PATCH 70/89] Fix as_pymongo bug --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 5fd84195..4c37d989 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -147,7 +147,7 @@ class BaseQuerySet(object): queryset._document._from_son(queryset._cursor[key], _auto_dereference=self._auto_dereference)) if queryset._as_pymongo: - return queryset._get_as_pymongo(queryset._cursor.next()) + return queryset._get_as_pymongo(queryset._cursor[key]) return queryset._document._from_son(queryset._cursor[key], _auto_dereference=self._auto_dereference) raise AttributeError From 4ee212e7d58a3fa39b5a4c18a311453b262a422c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 19:25:05 +0100 Subject: [PATCH 71/89] Skip Test due to server bug in 2.6 --- tests/queryset/geo.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py index 65ab519a..5148a48e 100644 --- a/tests/queryset/geo.py +++ b/tests/queryset/geo.py @@ -5,6 +5,8 @@ import unittest from datetime import datetime, timedelta from mongoengine import * +from nose.plugins.skip import SkipTest + __all__ = ("GeoQueriesTest",) @@ -139,6 +141,7 @@ class GeoQueriesTest(unittest.TestCase): def test_spherical_geospatial_operators(self): """Ensure that spherical geospatial queries are working """ + raise SkipTest("https://jira.mongodb.org/browse/SERVER-14039") class Point(Document): location = GeoPointField() From 3a0c69005ba274ec6d039b44c17af1c78fd48223 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 19:41:40 +0100 Subject: [PATCH 72/89] Update AUTHORS and Changelog Refs: #664, #677, #676, #673, #674, #655, #657, #626, #625, #619, #613, #608, #511, #559 --- AUTHORS | 5 +++++ docs/changelog.rst | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index c86df67c..82daf0d8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -192,3 +192,8 @@ that much better: * Jonathan Simon Prates (https://github.com/jonathansp) * Thiago Papageorgiou (https://github.com/tmpapageorgiou) * Omer Katz (https://github.com/thedrow) + * Falcon Dai (https://github.com/falcondai) + * Polyrabbit (https://github.com/polyrabbit) + * Sagiv Malihi (https://github.com/sagivmalihi) + * Dmitry Konishchev (https://github.com/KonishchevDmitry) + * \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index c980e904..88959617 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,10 +5,23 @@ Changelog Changes in 0.9.X - DEV ====================== -- pypy support #673 -- Enabled connection pooling #674 + - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x. - Removing support for Python < 2.6.6 +- Fixed $maxDistance location for geoJSON $near queries with MongoDB 2.6+ #664 +- QuerySet.modify() method to provide find_and_modify() like behaviour #677 +- Added support for the using() method on a queryset #676 +- PYPY support #673 +- Connection pooling #674 +- Avoid to open all documents from cursors in an if stmt #655 +- Ability to clear the ordering #657 +- Raise NotUniqueError in Document.update() on pymongo.errors.DuplicateKeyError #626 +- Slots - memory improvements #625 +- Fixed incorrectly split a query key when it ends with "_" #619 +- Geo docs updates #613 +- Workaround a dateutil bug #608 +- Conditional save for atomic-style operations #511 +- Allow dynamic dictionary-style field access #559 Changes in 0.8.7 ================ From c0a5b16a7f0d7f29be972f653e6d351e58be0b78 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 19:52:05 +0100 Subject: [PATCH 73/89] Travis bump --- AUTHORS | 1 - 1 file changed, 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 82daf0d8..2f832a3d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -196,4 +196,3 @@ that much better: * Polyrabbit (https://github.com/polyrabbit) * Sagiv Malihi (https://github.com/sagivmalihi) * Dmitry Konishchev (https://github.com/KonishchevDmitry) - * \ No newline at end of file From 5be5685a090b86b90609037257457824ab62a467 Mon Sep 17 00:00:00 2001 From: Martyn Smith Date: Fri, 27 Jun 2014 09:06:17 +0100 Subject: [PATCH 74/89] Test to illustrate failure in changed attribute tracking --- tests/document/delta.py | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/document/delta.py b/tests/document/delta.py index b0f5f01a..738dfa78 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -735,5 +735,47 @@ class DeltaTest(unittest.TestCase): mydoc._clear_changed_fields() self.assertEqual([], mydoc._get_changed_fields()) + def test_referenced_object_changed_attributes(self): + """Ensures that when you save a new reference to a field, the referenced object isn't altered""" + + class Organization(Document): + name = StringField() + + class User(Document): + name = StringField() + org = ReferenceField('Organization', required=True) + + Organization.drop_collection() + User.drop_collection() + + org1 = Organization(name='Org 1') + org1.save() + + org2 = Organization(name='Org 2') + org2.save() + + user = User(name='Fred', org=org1) + user.save() + + org1.reload() + org2.reload() + user.reload() + self.assertEqual(org1.name, 'Org 1') + self.assertEqual(org2.name, 'Org 2') + self.assertEqual(user.name, 'Fred') + + user.name = 'Harold' + user.org = org2 + + org2.name = 'New Org 2' + self.assertEqual(org2.name, 'New Org 2') + + user.save() + org2.save() + + self.assertEqual(org2.name, 'New Org 2') + org2.reload() + self.assertEqual(org2.name, 'New Org 2') + if __name__ == '__main__': unittest.main() From cd63865d311956f82396a5415f1f65f7d17cd654 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 09:08:07 +0100 Subject: [PATCH 75/89] Fix clear_changed_fields() clearing unsaved documents bug #602 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/base/document.py | 12 +++++++----- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index 2f832a3d..a0801b3c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -196,3 +196,4 @@ that much better: * Polyrabbit (https://github.com/polyrabbit) * Sagiv Malihi (https://github.com/sagivmalihi) * Dmitry Konishchev (https://github.com/KonishchevDmitry) + * Martyn Smith (https://github.com/martynsmith) diff --git a/docs/changelog.rst b/docs/changelog.rst index 88959617..5c3ac1c1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Fix clear_changed_fields() clearing unsaved documents bug #602 - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x. - Removing support for Python < 2.6.6 - Fixed $maxDistance location for geoJSON $near queries with MongoDB 2.6+ #664 diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 08401823..f6eec628 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -16,7 +16,7 @@ from mongoengine.errors import (ValidationError, InvalidDocumentError, from mongoengine.python_support import PY3, txt_type from mongoengine.base.common import get_document, ALLOW_INHERITANCE -from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict, SemiStrictDict +from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict, SemiStrictDict from mongoengine.base.fields import ComplexBaseField __all__ = ('BaseDocument', 'NON_FIELD_ERRORS') @@ -54,12 +54,12 @@ class BaseDocument(object): values[name] = value __auto_convert = values.pop("__auto_convert", True) signals.pre_init.send(self.__class__, document=self, values=values) - + if self.STRICT and not self._dynamic: self._data = StrictDict.create(allowed_keys=self._fields.keys())() else: self._data = SemiStrictDict.create(allowed_keys=self._fields.keys())() - + self._dynamic_fields = SON() # Assign default values to instance @@ -150,7 +150,7 @@ class BaseDocument(object): try: self__initialised = self._initialised except AttributeError: - self__initialised = False + self__initialised = False # Check if the user has created a new instance of a class if (self._is_document and self__initialised and self__created and name == self._meta['id_field']): @@ -407,6 +407,8 @@ class BaseDocument(object): else: data = getattr(data, part, None) if hasattr(data, "_changed_fields"): + if hasattr(data, "_is_document") and data._is_document: + continue data._changed_fields = [] self._changed_fields = [] @@ -596,7 +598,7 @@ class BaseDocument(object): msg = ("Invalid data to create a `%s` instance.\n%s" % (cls._class_name, errors)) raise InvalidDocumentError(msg) - + if cls.STRICT: data = dict((k, v) for k,v in data.iteritems() if k in cls._fields) obj = cls(__auto_convert=False, **data) From 72a051f2d3d4b09d19d1b0a47f35deff434ffc05 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 09:12:05 +0100 Subject: [PATCH 76/89] Update AUTHORS & Changelog #557 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index a0801b3c..339a58a3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -197,3 +197,4 @@ that much better: * Sagiv Malihi (https://github.com/sagivmalihi) * Dmitry Konishchev (https://github.com/KonishchevDmitry) * Martyn Smith (https://github.com/martynsmith) + * Andrei Zbikowski (https://github.com/b1naryth1ef) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5c3ac1c1..08fb24ef 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Fixes issue with recursive embedded document errors #557 - Fix clear_changed_fields() clearing unsaved documents bug #602 - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x. - Removing support for Python < 2.6.6 From 7f36ea55f57da2f159f5c15b08ada234bcffe38c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 09:14:56 +0100 Subject: [PATCH 77/89] Fix bulk test where behaviour changes based on mongo version --- tests/queryset/queryset.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index b97f9519..5b18dc63 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -651,7 +651,10 @@ class QuerySetTest(unittest.TestCase): blogs.append(Blog(title="post %s" % i, posts=[post1, post2])) Blog.objects.insert(blogs, load_bulk=False) - self.assertEqual(q, 99) # profiling logs each doc now :( + if (get_connection().max_wire_version <= 1): + self.assertEqual(q, 1) + else: + self.assertEqual(q, 99) # profiling logs each doc now in the bulk op Blog.drop_collection() Blog.ensure_indexes() @@ -660,7 +663,10 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(q, 0) Blog.objects.insert(blogs) - self.assertEqual(q, 100) # 99 or insert, and 1 for in bulk fetch + if (get_connection().max_wire_version <= 1): + self.assertEqual(q, 2) # 1 for insert, and 1 for in bulk fetch + else: + self.assertEqual(q, 100) # 99 for insert, and 1 for in bulk fetch Blog.drop_collection() From 0971ad0a80c24a7661fd7ce6a173379fb07d1b62 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 09:31:01 +0100 Subject: [PATCH 78/89] Update changelog & authors - #636 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 339a58a3..b9ab34df 100644 --- a/AUTHORS +++ b/AUTHORS @@ -198,3 +198,4 @@ that much better: * Dmitry Konishchev (https://github.com/KonishchevDmitry) * Martyn Smith (https://github.com/martynsmith) * Andrei Zbikowski (https://github.com/b1naryth1ef) + * Ronald van Rij (https://github.com/ronaldvanrij) diff --git a/docs/changelog.rst b/docs/changelog.rst index 08fb24ef..2ed0001b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Fix id shard key save issue #636 - Fixes issue with recursive embedded document errors #557 - Fix clear_changed_fields() clearing unsaved documents bug #602 - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x. From 5e776a07dd2a267017a3d91723ea7dbf1ab826a4 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Fri, 7 Mar 2014 16:08:06 -0800 Subject: [PATCH 79/89] allow ordering to be cleared --- mongoengine/queryset/base.py | 2 +- tests/queryset/queryset.py | 98 +++++++++++++++--------------------- 2 files changed, 42 insertions(+), 58 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 4c37d989..be5d66b0 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -192,7 +192,7 @@ class BaseQuerySet(object): .. versionadded:: 0.3 """ queryset = self.clone() - queryset = queryset.limit(2) + queryset = queryset.order_by().limit(2) queryset = queryset.filter(*q_objs, **query) try: diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 5b18dc63..4770c3e6 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -14,8 +14,8 @@ from pymongo.read_preferences import ReadPreference from bson import ObjectId from mongoengine import * -from mongoengine.connection import get_connection from mongoengine.python_support import PY3 +from mongoengine.connection import get_connection from mongoengine.context_managers import query_counter, switch_db from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, @@ -25,6 +25,12 @@ from mongoengine.errors import InvalidQueryError __all__ = ("QuerySetTest",) +class db_ops_tracker(query_counter): + def get_ops(self): + ignore_query = {"ns": {"$ne": "%s.system.indexes" % self.db.name}} + return list(self.db.system.profile.find(ignore_query)) + + class QuerySetTest(unittest.TestCase): def setUp(self): @@ -1048,74 +1054,52 @@ class QuerySetTest(unittest.TestCase): self.assertSequence(qs, expected) def test_clear_ordering(self): - """ Make sure one can clear the query set ordering by applying a - consecutive order_by() + """ Ensure that the default ordering can be cleared by calling order_by(). """ + class BlogPost(Document): + title = StringField() + published_date = DateTimeField() - class Person(Document): - name = StringField() - - Person.drop_collection() - Person(name="A").save() - Person(name="B").save() - - qs = Person.objects.order_by('-name') - - # Make sure we can clear a previously specified ordering - with query_counter() as q: - lst = list(qs.order_by()) - - op = q.db.system.profile.find({"ns": - {"$ne": "%s.system.indexes" % q.db.name}})[0] - - self.assertTrue('$orderby' not in op['query']) - self.assertEqual(lst[0].name, 'A') - - # Make sure previously specified ordering is preserved during - # consecutive calls to the same query set - with query_counter() as q: - lst = list(qs) - - op = q.db.system.profile.find({"ns": - {"$ne": "%s.system.indexes" % q.db.name}})[0] - - self.assertTrue('$orderby' in op['query']) - self.assertEqual(lst[0].name, 'B') - - def test_clear_default_ordering(self): - - class Person(Document): - name = StringField() meta = { - 'ordering': ['-name'] + 'ordering': ['-published_date'] } - Person.drop_collection() - Person(name="A").save() - Person(name="B").save() + BlogPost.drop_collection() - qs = Person.objects + with db_ops_tracker() as q: + BlogPost.objects.filter(title='whatever').first() + self.assertEqual(len(q.get_ops()), 1) + self.assertEqual(q.get_ops()[0]['query']['$orderby'], {u'published_date': -1}) - # Make sure clearing default ordering works - with query_counter() as q: - lst = list(qs.order_by()) + with db_ops_tracker() as q: + BlogPost.objects.filter(title='whatever').order_by().first() + self.assertEqual(len(q.get_ops()), 1) + print q.get_ops()[0]['query'] + self.assertFalse('$orderby' in q.get_ops()[0]['query']) - op = q.db.system.profile.find({"ns": - {"$ne": "%s.system.indexes" % q.db.name}})[0] + def test_no_ordering_for_get(self): + """ Ensure that Doc.objects.get doesn't use any ordering. + """ + class BlogPost(Document): + title = StringField() + published_date = DateTimeField() - self.assertTrue('$orderby' not in op['query']) - self.assertEqual(lst[0].name, 'A') + meta = { + 'ordering': ['-published_date'] + } - # Make sure default ordering is preserved during consecutive calls - # to the same query set - with query_counter() as q: - lst = list(qs) + BlogPost.objects.create(title='whatever', published_date=datetime.utcnow()) - op = q.db.system.profile.find({"ns": - {"$ne": "%s.system.indexes" % q.db.name}})[0] + with db_ops_tracker() as q: + BlogPost.objects.get(title='whatever') + self.assertEqual(len(q.get_ops()), 1) + self.assertFalse('$orderby' in q.get_ops()[0]['query']) - self.assertTrue('$orderby' in op['query']) - self.assertEqual(lst[0].name, 'B') + # Ordering should be ignored for .get even if we set it explicitly + with db_ops_tracker() as q: + BlogPost.objects.order_by('-title').get(title='whatever') + self.assertEqual(len(q.get_ops()), 1) + self.assertFalse('$orderby' in q.get_ops()[0]['query']) def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. From 2e4fb86b866fa0c9fe63c499a25b2f51935d9db3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 10:00:16 +0100 Subject: [PATCH 80/89] Don't query with $orderby for qs.get() #600 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2ed0001b..45f7e020 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Don't query with $orderby for qs.get() #600 - Fix id shard key save issue #636 - Fixes issue with recursive embedded document errors #557 - Fix clear_changed_fields() clearing unsaved documents bug #602 From 7013033ae415be314c6e7de91b6ea2bd321e06d7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 10:03:35 +0100 Subject: [PATCH 81/89] Update changelog & AUTHORS #594 #589 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index b9ab34df..c6c47d79 100644 --- a/AUTHORS +++ b/AUTHORS @@ -199,3 +199,4 @@ that much better: * Martyn Smith (https://github.com/martynsmith) * Andrei Zbikowski (https://github.com/b1naryth1ef) * Ronald van Rij (https://github.com/ronaldvanrij) + * François Schmidts (https://github.com/jaesivsm) diff --git a/docs/changelog.rst b/docs/changelog.rst index 45f7e020..c722b592 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- post_save signal now has access to delta information about field changes #594 #589 - Don't query with $orderby for qs.get() #600 - Fix id shard key save issue #636 - Fixes issue with recursive embedded document errors #557 From 67eaf120b916f5780b9506904185de65dc1872ce Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 10:07:05 +0100 Subject: [PATCH 82/89] db_alias support and fixes for custom map/reduce output #586 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c722b592..9b524c2d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Added db_alias support and fixes for custom map/reduce output #586 - post_save signal now has access to delta information about field changes #594 #589 - Don't query with $orderby for qs.get() #600 - Fix id shard key save issue #636 From 9cc4359c0468fdb99ddd1d9a01f2269dc3ee16e6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 11:10:14 +0100 Subject: [PATCH 83/89] Added the ability to reload specific document fields #100 --- docs/changelog.rst | 1 + mongoengine/document.py | 26 +++-- tests/document/instance.py | 212 ++++++++++++++++++++----------------- 3 files changed, 135 insertions(+), 104 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9b524c2d..13cd1b69 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Added the ability to reload specific document fields #100 - Added db_alias support and fixes for custom map/reduce output #586 - post_save signal now has access to delta information about field changes #594 #589 - Don't query with $orderby for qs.get() #600 diff --git a/mongoengine/document.py b/mongoengine/document.py index 7541ee57..67f442ae 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -54,7 +54,7 @@ class EmbeddedDocument(BaseDocument): `_cls` set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` dictionary. """ - + __slots__ = ('_instance') # The __metaclass__ attribute is removed by 2to3 when running with Python3 @@ -463,27 +463,41 @@ class Document(BaseDocument): DeReference()([self], max_depth + 1) return self - def reload(self, max_depth=1): + def reload(self, *fields, **kwargs): """Reloads all attributes from the database. + :param fields: (optional) args list of fields to reload + :param max_depth: (optional) depth of dereferencing to follow + .. versionadded:: 0.1.2 .. versionchanged:: 0.6 Now chainable + .. versionchanged:: 0.9 Can provide specific fields to reload """ + max_depth = 1 + if fields and isinstance(fields[0], int): + max_depth = fields[0] + fields = fields[1:] + elif "max_depth" in kwargs: + max_depth = kwargs["max_depth"] + if not self.pk: raise self.DoesNotExist("Document does not exist") obj = self._qs.read_preference(ReadPreference.PRIMARY).filter( - **self._object_key).limit(1).select_related(max_depth=max_depth) - + **self._object_key).only(*fields).limit(1 + ).select_related(max_depth=max_depth) if obj: obj = obj[0] else: raise self.DoesNotExist("Document does not exist") + for field in self._fields_ordered: - setattr(self, field, self._reload(field, obj[field])) + if not fields or field in fields: + setattr(self, field, self._reload(field, obj[field])) + self._changed_fields = obj._changed_fields self._created = False - return obj + return self def _reload(self, key, value): """Used by :meth:`~mongoengine.Document.reload` to ensure the diff --git a/tests/document/instance.py b/tests/document/instance.py index 54758955..81ecd29b 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -50,7 +50,7 @@ class InstanceTest(unittest.TestCase): continue self.db.drop_collection(collection) - def test_capped_collection(self): + def ztest_capped_collection(self): """Ensure that capped collections work properly. """ class Log(Document): @@ -90,7 +90,7 @@ class InstanceTest(unittest.TestCase): Log.drop_collection() - def test_repr(self): + def ztest_repr(self): """Ensure that unicode representation works """ class Article(Document): @@ -103,7 +103,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual('', repr(doc)) - def test_queryset_resurrects_dropped_collection(self): + def ztest_queryset_resurrects_dropped_collection(self): self.Person.drop_collection() self.assertEqual([], list(self.Person.objects())) @@ -116,7 +116,7 @@ class InstanceTest(unittest.TestCase): self.Person.drop_collection() self.assertEqual([], list(Actor.objects())) - def test_polymorphic_references(self): + def ztest_polymorphic_references(self): """Ensure that the correct subclasses are returned from a query when using references / generic references """ @@ -163,7 +163,7 @@ class InstanceTest(unittest.TestCase): Zoo.drop_collection() Animal.drop_collection() - def test_reference_inheritance(self): + def ztest_reference_inheritance(self): class Stats(Document): created = DateTimeField(default=datetime.now) @@ -188,7 +188,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(list_stats, CompareStats.objects.first().stats) - def test_db_field_load(self): + def ztest_db_field_load(self): """Ensure we load data correctly """ class Person(Document): @@ -208,7 +208,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") self.assertEqual(Person.objects.get(name="Fred").rank, "Private") - def test_db_embedded_doc_field_load(self): + def ztest_db_embedded_doc_field_load(self): """Ensure we load embedded document data correctly """ class Rank(EmbeddedDocument): @@ -234,7 +234,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") self.assertEqual(Person.objects.get(name="Fred").rank, "Private") - def test_custom_id_field(self): + def ztest_custom_id_field(self): """Ensure that documents may be created with custom primary keys. """ class User(Document): @@ -286,7 +286,7 @@ class InstanceTest(unittest.TestCase): User.drop_collection() - def test_document_not_registered(self): + def ztest_document_not_registered(self): class Place(Document): name = StringField() @@ -310,7 +310,7 @@ class InstanceTest(unittest.TestCase): print Place.objects.all() self.assertRaises(NotRegistered, query_without_importing_nice_place) - def test_document_registry_regressions(self): + def ztest_document_registry_regressions(self): class Location(Document): name = StringField() @@ -324,14 +324,14 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Area, get_document("Area")) self.assertEqual(Area, get_document("Location.Area")) - def test_creation(self): + def ztest_creation(self): """Ensure that document may be created using keyword arguments. """ person = self.Person(name="Test User", age=30) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 30) - def test_to_dbref(self): + def ztest_to_dbref(self): """Ensure that you can get a dbref of a document""" person = self.Person(name="Test User", age=30) self.assertRaises(OperationError, person.to_dbref) @@ -353,11 +353,19 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 20) + person.reload('age') + self.assertEqual(person.name, "Test User") + self.assertEqual(person.age, 21) + person.reload() self.assertEqual(person.name, "Mr Test User") self.assertEqual(person.age, 21) - def test_reload_sharded(self): + person.reload() + self.assertEqual(person.name, "Mr Test User") + self.assertEqual(person.age, 21) + + def ztest_reload_sharded(self): class Animal(Document): superphylum = StringField() meta = {'shard_key': ('superphylum',)} @@ -368,7 +376,7 @@ class InstanceTest(unittest.TestCase): doc.reload() Animal.drop_collection() - def test_reload_referencing(self): + def ztest_reload_referencing(self): """Ensures reloading updates weakrefs correctly """ class Embedded(EmbeddedDocument): @@ -402,6 +410,7 @@ class InstanceTest(unittest.TestCase): 'embedded_field.dict_field']) doc.save() + self.assertEqual(len(doc.list_field), 4) doc = doc.reload(10) self.assertEqual(doc._get_changed_fields(), []) self.assertEqual(len(doc.list_field), 4) @@ -409,7 +418,17 @@ class InstanceTest(unittest.TestCase): self.assertEqual(len(doc.embedded_field.list_field), 4) self.assertEqual(len(doc.embedded_field.dict_field), 2) - def test_reload_doesnt_exist(self): + doc.list_field.append(1) + doc.save() + doc.dict_field['extra'] = 1 + doc = doc.reload(10, 'list_field') + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(len(doc.list_field), 5) + self.assertEqual(len(doc.dict_field), 3) + self.assertEqual(len(doc.embedded_field.list_field), 4) + self.assertEqual(len(doc.embedded_field.dict_field), 2) + + def ztest_reload_doesnt_exist(self): class Foo(Document): pass @@ -430,7 +449,7 @@ class InstanceTest(unittest.TestCase): except Exception as ex: self.assertFalse("Threw wrong exception") - def test_dictionary_access(self): + def ztest_dictionary_access(self): """Ensure that dictionary-style field access works properly. """ person = self.Person(name='Test User', age=30) @@ -450,7 +469,7 @@ class InstanceTest(unittest.TestCase): self.assertFalse('age' in person) self.assertFalse('nationality' in person) - def test_embedded_document_to_mongo(self): + def ztest_embedded_document_to_mongo(self): class Person(EmbeddedDocument): name = StringField() age = IntField() @@ -465,14 +484,14 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Employee(name="Bob", age=35, salary=0).to_mongo().keys(), ['_cls', 'name', 'age', 'salary']) - def test_embedded_document_to_mongo_id(self): + def ztest_embedded_document_to_mongo_id(self): class SubDoc(EmbeddedDocument): id = StringField(required=True) sub_doc = SubDoc(id="abc") self.assertEqual(sub_doc.to_mongo().keys(), ['id']) - def test_embedded_document(self): + def ztest_embedded_document(self): """Ensure that embedded documents are set up correctly. """ class Comment(EmbeddedDocument): @@ -481,7 +500,7 @@ class InstanceTest(unittest.TestCase): self.assertTrue('content' in Comment._fields) self.assertFalse('id' in Comment._fields) - def test_embedded_document_instance(self): + def ztest_embedded_document_instance(self): """Ensure that embedded documents can reference parent instance """ class Embedded(EmbeddedDocument): @@ -496,7 +515,7 @@ class InstanceTest(unittest.TestCase): doc = Doc.objects.get() self.assertEqual(doc, doc.embedded_field._instance) - def test_embedded_document_complex_instance(self): + def ztest_embedded_document_complex_instance(self): """Ensure that embedded documents in complex fields can reference parent instance""" class Embedded(EmbeddedDocument): @@ -511,13 +530,10 @@ class InstanceTest(unittest.TestCase): doc = Doc.objects.get() self.assertEqual(doc, doc.embedded_field[0]._instance) - def test_instance_is_set_on_setattr(self): + def ztest_instance_is_set_on_setattr(self): class Email(EmbeddedDocument): email = EmailField() - def clean(self): - print "instance:" - print self._instance class Account(Document): email = EmbeddedDocumentField(Email) @@ -531,7 +547,7 @@ class InstanceTest(unittest.TestCase): acc1 = Account.objects.first() self.assertTrue(hasattr(acc1._data["email"], "_instance")) - def test_document_clean(self): + def ztest_document_clean(self): class TestDocument(Document): status = StringField() pub_date = DateTimeField() @@ -565,7 +581,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(type(t.pub_date), datetime) - def test_document_embedded_clean(self): + def ztest_document_embedded_clean(self): class TestEmbeddedDocument(EmbeddedDocument): x = IntField(required=True) y = IntField(required=True) @@ -601,7 +617,7 @@ class InstanceTest(unittest.TestCase): t = TestDocument(doc=TestEmbeddedDocument(x=15, y=35, z=5)) t.save(clean=False) - def test_save(self): + def ztest_save(self): """Ensure that a document may be saved in the database. """ # Create person object and save it to the database @@ -626,7 +642,7 @@ class InstanceTest(unittest.TestCase): except ValidationError: self.fail() - def test_save_to_a_value_that_equates_to_false(self): + def ztest_save_to_a_value_that_equates_to_false(self): class Thing(EmbeddedDocument): count = IntField() @@ -646,7 +662,7 @@ class InstanceTest(unittest.TestCase): user.reload() self.assertEqual(user.thing.count, 0) - def test_save_max_recursion_not_hit(self): + def ztest_save_max_recursion_not_hit(self): class Person(Document): name = StringField() @@ -672,7 +688,7 @@ class InstanceTest(unittest.TestCase): p0.name = 'wpjunior' p0.save() - def test_save_max_recursion_not_hit_with_file_field(self): + def ztest_save_max_recursion_not_hit_with_file_field(self): class Foo(Document): name = StringField() @@ -696,7 +712,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(b.picture, b.bar.picture, b.bar.bar.picture) - def test_save_cascades(self): + def ztest_save_cascades(self): class Person(Document): name = StringField() @@ -719,7 +735,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) - def test_save_cascade_kwargs(self): + def ztest_save_cascade_kwargs(self): class Person(Document): name = StringField() @@ -740,7 +756,7 @@ class InstanceTest(unittest.TestCase): p2.reload() self.assertEqual(p1.name, p2.parent.name) - def test_save_cascade_meta_false(self): + def ztest_save_cascade_meta_false(self): class Person(Document): name = StringField() @@ -769,7 +785,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) - def test_save_cascade_meta_true(self): + def ztest_save_cascade_meta_true(self): class Person(Document): name = StringField() @@ -794,7 +810,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertNotEqual(p1.name, p.parent.name) - def test_save_cascades_generically(self): + def ztest_save_cascades_generically(self): class Person(Document): name = StringField() @@ -820,7 +836,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) - def test_save_atomicity_condition(self): + def ztest_save_atomicity_condition(self): class Widget(Document): toggle = BooleanField(default=False) @@ -835,7 +851,7 @@ class InstanceTest(unittest.TestCase): return uuid.UUID(int=i) Widget.drop_collection() - + w1 = Widget(toggle=False, save_id=UUID(1)) # ignore save_condition on new record creation @@ -893,8 +909,8 @@ class InstanceTest(unittest.TestCase): w1.reload() self.assertTrue(w1.toggle) self.assertEqual(w1.count, 3) - - def test_update(self): + + def ztest_update(self): """Ensure that an existing document is updated instead of be overwritten.""" # Create person object and save it to the database @@ -978,7 +994,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.name, None) self.assertEqual(person.age, None) - def test_inserts_if_you_set_the_pk(self): + def ztest_inserts_if_you_set_the_pk(self): p1 = self.Person(name='p1', id=bson.ObjectId()).save() p2 = self.Person(name='p2') p2.id = bson.ObjectId() @@ -986,7 +1002,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(2, self.Person.objects.count()) - def test_can_save_if_not_included(self): + def ztest_can_save_if_not_included(self): class EmbeddedDoc(EmbeddedDocument): pass @@ -1035,7 +1051,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(my_doc.string_field, "string") self.assertEqual(my_doc.int_field, 1) - def test_document_update(self): + def ztest_document_update(self): def update_not_saved_raises(): person = self.Person(name='dcrosta') @@ -1064,7 +1080,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidQueryError, update_no_op_raises) - def test_update_unique_field(self): + def ztest_update_unique_field(self): class Doc(Document): name = StringField(unique=True) @@ -1074,7 +1090,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(NotUniqueError, lambda: doc2.update(set__name=doc1.name)) - def test_embedded_update(self): + def ztest_embedded_update(self): """ Test update on `EmbeddedDocumentField` fields """ @@ -1098,7 +1114,7 @@ class InstanceTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") - def test_embedded_update_db_field(self): + def ztest_embedded_update_db_field(self): """ Test update on `EmbeddedDocumentField` fields when db_field is other than default. @@ -1125,7 +1141,7 @@ class InstanceTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") - def test_save_only_changed_fields(self): + def ztest_save_only_changed_fields(self): """Ensure save only sets / unsets changed fields """ @@ -1154,7 +1170,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.age, 21) self.assertEqual(person.active, False) - def test_query_count_when_saving(self): + def ztest_query_count_when_saving(self): """Ensure references don't cause extra fetches when saving""" class Organization(Document): name = StringField() @@ -1247,7 +1263,7 @@ class InstanceTest(unittest.TestCase): sub.save() self.assertEqual(q, 3) - def test_set_unset_one_operation(self): + def ztest_set_unset_one_operation(self): """Ensure that $set and $unset actions are performed in the same operation. """ @@ -1269,7 +1285,7 @@ class InstanceTest(unittest.TestCase): foo.save() self.assertEqual(1, q) - def test_save_only_changed_fields_recursive(self): + def ztest_save_only_changed_fields_recursive(self): """Ensure save only sets / unsets changed fields """ @@ -1311,7 +1327,7 @@ class InstanceTest(unittest.TestCase): person = self.Person.objects.get() self.assertFalse(person.comments_dict['first_post'].published) - def test_delete(self): + def ztest_delete(self): """Ensure that document may be deleted using the delete method. """ person = self.Person(name="Test User", age=30) @@ -1320,7 +1336,7 @@ class InstanceTest(unittest.TestCase): person.delete() self.assertEqual(self.Person.objects.count(), 0) - def test_save_custom_id(self): + def ztest_save_custom_id(self): """Ensure that a document may be saved with a custom _id. """ # Create person object and save it to the database @@ -1332,7 +1348,7 @@ class InstanceTest(unittest.TestCase): person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') - def test_save_custom_pk(self): + def ztest_save_custom_pk(self): """Ensure that a document may be saved with a custom _id using pk alias. """ # Create person object and save it to the database @@ -1344,7 +1360,7 @@ class InstanceTest(unittest.TestCase): person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') - def test_save_list(self): + def ztest_save_list(self): """Ensure that a list field may be properly saved. """ class Comment(EmbeddedDocument): @@ -1371,7 +1387,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - def test_list_search_by_embedded(self): + def ztest_list_search_by_embedded(self): class User(Document): username = StringField(required=True) @@ -1423,7 +1439,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual([p1, p2, p4], list(Page.objects.filter(comments__user=u2))) self.assertEqual([p1, p3], list(Page.objects.filter(comments__user=u3))) - def test_save_embedded_document(self): + def ztest_save_embedded_document(self): """Ensure that a document with an embedded document field may be saved in the database. """ @@ -1447,7 +1463,7 @@ class InstanceTest(unittest.TestCase): # Ensure that the 'details' embedded object saved correctly self.assertEqual(employee_obj['details']['position'], 'Developer') - def test_embedded_update_after_save(self): + def ztest_embedded_update_after_save(self): """ Test update of `EmbeddedDocumentField` attached to a newly saved document. @@ -1470,7 +1486,7 @@ class InstanceTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") - def test_updating_an_embedded_document(self): + def ztest_updating_an_embedded_document(self): """Ensure that a document with an embedded document field may be saved in the database. """ @@ -1505,7 +1521,7 @@ class InstanceTest(unittest.TestCase): promoted_employee.reload() self.assertEqual(promoted_employee.details, None) - def test_object_mixins(self): + def ztest_object_mixins(self): class NameMixin(object): name = StringField() @@ -1520,7 +1536,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(['id', 'name', 'widgets'], sorted(Bar._fields.keys())) - def test_mixin_inheritance(self): + def ztest_mixin_inheritance(self): class BaseMixIn(object): count = IntField() data = StringField() @@ -1544,7 +1560,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(t.data, "test") self.assertEqual(t.count, 12) - def test_save_reference(self): + def ztest_save_reference(self): """Ensure that a document reference field may be saved in the database. """ @@ -1580,7 +1596,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - def test_duplicate_db_fields_raise_invalid_document_error(self): + def ztest_duplicate_db_fields_raise_invalid_document_error(self): """Ensure a InvalidDocumentError is thrown if duplicate fields declare the same db_field""" @@ -1591,7 +1607,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, throw_invalid_document_error) - def test_invalid_son(self): + def ztest_invalid_son(self): """Raise an error if loading invalid data""" class Occurrence(EmbeddedDocument): number = IntField() @@ -1608,7 +1624,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, raise_invalid_document) - def test_reverse_delete_rule_cascade_and_nullify(self): + def ztest_reverse_delete_rule_cascade_and_nullify(self): """Ensure that a referenced document is also deleted upon deletion. """ @@ -1639,7 +1655,7 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(BlogPost.objects.count(), 0) - def test_reverse_delete_rule_with_document_inheritance(self): + def ztest_reverse_delete_rule_with_document_inheritance(self): """Ensure that a referenced document is also deleted upon deletion of a child document. """ @@ -1674,7 +1690,7 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(BlogPost.objects.count(), 0) - def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): + def ztest_reverse_delete_rule_cascade_and_nullify_complex_field(self): """Ensure that a referenced document is also deleted upon deletion for complex fields. """ @@ -1708,7 +1724,7 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(BlogPost.objects.count(), 0) - def test_reverse_delete_rule_cascade_triggers_pre_delete_signal(self): + def ztest_reverse_delete_rule_cascade_triggers_pre_delete_signal(self): ''' ensure the pre_delete signal is triggered upon a cascading deletion setup a blog post with content, an author and editor delete the author which triggers deletion of blogpost via cascade @@ -1744,7 +1760,7 @@ class InstanceTest(unittest.TestCase): editor = Editor.objects(name='Max P.').get() self.assertEqual(editor.review_queue, 0) - def test_two_way_reverse_delete_rule(self): + def ztest_two_way_reverse_delete_rule(self): """Ensure that Bi-Directional relationships work with reverse_delete_rule """ @@ -1777,7 +1793,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Bar.objects.count(), 1) # No effect on the BlogPost self.assertEqual(Bar.objects.get().foo, None) - def test_invalid_reverse_delete_rules_raise_errors(self): + def ztest_invalid_reverse_delete_rules_raise_errors(self): def throw_invalid_document_error(): class Blog(Document): @@ -1794,7 +1810,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, throw_invalid_document_error_embedded) - def test_reverse_delete_rule_cascade_recurs(self): + def ztest_reverse_delete_rule_cascade_recurs(self): """Ensure that a chain of documents is also deleted upon cascaded deletion. """ @@ -1831,7 +1847,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() Comment.drop_collection() - def test_reverse_delete_rule_deny(self): + def ztest_reverse_delete_rule_deny(self): """Ensure that a document cannot be referenced if there are still documents referring to it. """ @@ -1886,7 +1902,7 @@ class InstanceTest(unittest.TestCase): A.drop_collection() B.drop_collection() - def test_document_hash(self): + def ztest_document_hash(self): """Test document in list, dict, set """ class User(Document): @@ -1934,7 +1950,7 @@ class InstanceTest(unittest.TestCase): self.assertTrue(u1 in all_user_set) - def test_picklable(self): + def ztest_picklable(self): pickle_doc = PickleTest(number=1, string="One", lists=['1', '2']) pickle_doc.embedded = PickleEmbedded() @@ -1960,7 +1976,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(pickle_doc.string, "Two") self.assertEqual(pickle_doc.lists, ["1", "2", "3"]) - def test_dynamic_document_pickle(self): + def ztest_dynamic_document_pickle(self): pickle_doc = PickleDynamicTest(name="test", number=1, string="One", lists=['1', '2']) pickle_doc.embedded = PickleDyanmicEmbedded(foo="Bar") @@ -1983,13 +1999,13 @@ class InstanceTest(unittest.TestCase): self.assertEqual(resurrected.embedded._dynamic_fields.keys(), pickle_doc.embedded._dynamic_fields.keys()) - def test_picklable_on_signals(self): + def ztest_picklable_on_signals(self): pickle_doc = PickleSignalsTest(number=1, string="One", lists=['1', '2']) pickle_doc.embedded = PickleEmbedded() pickle_doc.save() pickle_doc.delete() - def test_throw_invalid_document_error(self): + def ztest_throw_invalid_document_error(self): # test handles people trying to upsert def throw_invalid_document_error(): @@ -1998,7 +2014,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, throw_invalid_document_error) - def test_mutating_documents(self): + def ztest_mutating_documents(self): class B(EmbeddedDocument): field1 = StringField(default='field1') @@ -2029,7 +2045,7 @@ class InstanceTest(unittest.TestCase): a.reload() self.assertEqual(a.b.field2.c_field, 'new value') - def test_can_save_false_values(self): + def ztest_can_save_false_values(self): """Ensures you can save False values on save""" class Doc(Document): foo = StringField() @@ -2043,7 +2059,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Doc.objects(archived=False).count(), 1) - def test_can_save_false_values_dynamic(self): + def ztest_can_save_false_values_dynamic(self): """Ensures you can save False values on dynamic docs""" class Doc(DynamicDocument): foo = StringField() @@ -2056,7 +2072,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Doc.objects(archived=False).count(), 1) - def test_do_not_save_unchanged_references(self): + def ztest_do_not_save_unchanged_references(self): """Ensures cascading saves dont auto update""" class Job(Document): name = StringField() @@ -2087,7 +2103,7 @@ class InstanceTest(unittest.TestCase): finally: Collection.update = orig_update - def test_db_alias_tests(self): + def ztest_db_alias_tests(self): """ DB Alias tests """ # mongoenginetest - Is default connection alias from setUp() # Register Aliases @@ -2143,7 +2159,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Book._get_collection(), get_db("testdb-2")[Book._get_collection_name()]) self.assertEqual(AuthorBooks._get_collection(), get_db("testdb-3")[AuthorBooks._get_collection_name()]) - def test_db_alias_overrides(self): + def ztest_db_alias_overrides(self): """db_alias can be overriden """ # Register a connection with db_alias testdb-2 @@ -2168,7 +2184,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual('mongoenginetest2', B._get_collection().database.name) - def test_db_alias_propagates(self): + def ztest_db_alias_propagates(self): """db_alias propagates? """ register_connection('testdb-1', 'mongoenginetest2') @@ -2182,7 +2198,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual('testdb-1', B._meta.get('db_alias')) - def test_db_ref_usage(self): + def ztest_db_ref_usage(self): """ DB Ref usage in dict_fields""" class User(Document): @@ -2260,7 +2276,7 @@ class InstanceTest(unittest.TestCase): })]), "1,2") - def test_switch_db_instance(self): + def ztest_switch_db_instance(self): register_connection('testdb-1', 'mongoenginetest2') class Group(Document): @@ -2310,7 +2326,7 @@ class InstanceTest(unittest.TestCase): group = Group.objects.first() self.assertEqual("hello - default", group.name) - def test_no_overwritting_no_data_loss(self): + def ztest_no_overwritting_no_data_loss(self): class User(Document): username = StringField(primary_key=True) @@ -2334,7 +2350,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual("Bar", user._data["foo"]) self.assertEqual([1, 2, 3], user._data["data"]) - def test_spaces_in_keys(self): + def ztest_spaces_in_keys(self): class Embedded(DynamicEmbeddedDocument): pass @@ -2350,7 +2366,7 @@ class InstanceTest(unittest.TestCase): one = Doc.objects.filter(**{'hello world': 1}).count() self.assertEqual(1, one) - def test_shard_key(self): + def ztest_shard_key(self): class LogEntry(Document): machine = StringField() log = StringField() @@ -2375,7 +2391,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) - def test_shard_key_primary(self): + def ztest_shard_key_primary(self): class LogEntry(Document): machine = StringField(primary_key=True) log = StringField() @@ -2400,7 +2416,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) - def test_kwargs_simple(self): + def ztest_kwargs_simple(self): class Embedded(EmbeddedDocument): name = StringField() @@ -2416,7 +2432,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(classic_doc, dict_doc) self.assertEqual(classic_doc._data, dict_doc._data) - def test_kwargs_complex(self): + def ztest_kwargs_complex(self): class Embedded(EmbeddedDocument): name = StringField() @@ -2435,21 +2451,21 @@ class InstanceTest(unittest.TestCase): self.assertEqual(classic_doc, dict_doc) self.assertEqual(classic_doc._data, dict_doc._data) - def test_positional_creation(self): + def ztest_positional_creation(self): """Ensure that document may be created using positional arguments. """ person = self.Person("Test User", 42) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - def test_mixed_creation(self): + def ztest_mixed_creation(self): """Ensure that document may be created using mixed arguments. """ person = self.Person("Test User", age=42) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - def test_mixed_creation_dynamic(self): + def ztest_mixed_creation_dynamic(self): """Ensure that document may be created using mixed arguments. """ class Person(DynamicDocument): @@ -2459,7 +2475,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - def test_bad_mixed_creation(self): + def ztest_bad_mixed_creation(self): """Ensure that document gives correct error when duplicating arguments """ def construct_bad_instance(): @@ -2467,7 +2483,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(TypeError, construct_bad_instance) - def test_data_contains_id_field(self): + def ztest_data_contains_id_field(self): """Ensure that asking for _data returns 'id' """ class Person(Document): @@ -2480,7 +2496,7 @@ class InstanceTest(unittest.TestCase): self.assertTrue('id' in person._data.keys()) self.assertEqual(person._data.get('id'), person.id) - def test_complex_nesting_document_and_embedded_document(self): + def ztest_complex_nesting_document_and_embedded_document(self): class Macro(EmbeddedDocument): value = DynamicField(default="UNDEFINED") @@ -2521,7 +2537,7 @@ class InstanceTest(unittest.TestCase): system = NodesSystem.objects.first() self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value) - def test_embedded_document_equality(self): + def ztest_embedded_document_equality(self): class Test(Document): field = StringField(required=True) From de9ba12779c6e1b79ec0719463466f17c405d387 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 11:16:23 +0100 Subject: [PATCH 84/89] Turn on tests --- tests/document/instance.py | 186 ++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 81ecd29b..951871b6 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -50,7 +50,7 @@ class InstanceTest(unittest.TestCase): continue self.db.drop_collection(collection) - def ztest_capped_collection(self): + def test_capped_collection(self): """Ensure that capped collections work properly. """ class Log(Document): @@ -90,7 +90,7 @@ class InstanceTest(unittest.TestCase): Log.drop_collection() - def ztest_repr(self): + def test_repr(self): """Ensure that unicode representation works """ class Article(Document): @@ -103,7 +103,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual('', repr(doc)) - def ztest_queryset_resurrects_dropped_collection(self): + def test_queryset_resurrects_dropped_collection(self): self.Person.drop_collection() self.assertEqual([], list(self.Person.objects())) @@ -116,7 +116,7 @@ class InstanceTest(unittest.TestCase): self.Person.drop_collection() self.assertEqual([], list(Actor.objects())) - def ztest_polymorphic_references(self): + def test_polymorphic_references(self): """Ensure that the correct subclasses are returned from a query when using references / generic references """ @@ -163,7 +163,7 @@ class InstanceTest(unittest.TestCase): Zoo.drop_collection() Animal.drop_collection() - def ztest_reference_inheritance(self): + def test_reference_inheritance(self): class Stats(Document): created = DateTimeField(default=datetime.now) @@ -188,7 +188,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(list_stats, CompareStats.objects.first().stats) - def ztest_db_field_load(self): + def test_db_field_load(self): """Ensure we load data correctly """ class Person(Document): @@ -208,7 +208,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") self.assertEqual(Person.objects.get(name="Fred").rank, "Private") - def ztest_db_embedded_doc_field_load(self): + def test_db_embedded_doc_field_load(self): """Ensure we load embedded document data correctly """ class Rank(EmbeddedDocument): @@ -234,7 +234,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") self.assertEqual(Person.objects.get(name="Fred").rank, "Private") - def ztest_custom_id_field(self): + def test_custom_id_field(self): """Ensure that documents may be created with custom primary keys. """ class User(Document): @@ -286,7 +286,7 @@ class InstanceTest(unittest.TestCase): User.drop_collection() - def ztest_document_not_registered(self): + def test_document_not_registered(self): class Place(Document): name = StringField() @@ -310,7 +310,7 @@ class InstanceTest(unittest.TestCase): print Place.objects.all() self.assertRaises(NotRegistered, query_without_importing_nice_place) - def ztest_document_registry_regressions(self): + def test_document_registry_regressions(self): class Location(Document): name = StringField() @@ -324,14 +324,14 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Area, get_document("Area")) self.assertEqual(Area, get_document("Location.Area")) - def ztest_creation(self): + def test_creation(self): """Ensure that document may be created using keyword arguments. """ person = self.Person(name="Test User", age=30) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 30) - def ztest_to_dbref(self): + def test_to_dbref(self): """Ensure that you can get a dbref of a document""" person = self.Person(name="Test User", age=30) self.assertRaises(OperationError, person.to_dbref) @@ -365,7 +365,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.name, "Mr Test User") self.assertEqual(person.age, 21) - def ztest_reload_sharded(self): + def test_reload_sharded(self): class Animal(Document): superphylum = StringField() meta = {'shard_key': ('superphylum',)} @@ -376,7 +376,7 @@ class InstanceTest(unittest.TestCase): doc.reload() Animal.drop_collection() - def ztest_reload_referencing(self): + def test_reload_referencing(self): """Ensures reloading updates weakrefs correctly """ class Embedded(EmbeddedDocument): @@ -428,7 +428,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(len(doc.embedded_field.list_field), 4) self.assertEqual(len(doc.embedded_field.dict_field), 2) - def ztest_reload_doesnt_exist(self): + def test_reload_doesnt_exist(self): class Foo(Document): pass @@ -449,7 +449,7 @@ class InstanceTest(unittest.TestCase): except Exception as ex: self.assertFalse("Threw wrong exception") - def ztest_dictionary_access(self): + def test_dictionary_access(self): """Ensure that dictionary-style field access works properly. """ person = self.Person(name='Test User', age=30) @@ -469,7 +469,7 @@ class InstanceTest(unittest.TestCase): self.assertFalse('age' in person) self.assertFalse('nationality' in person) - def ztest_embedded_document_to_mongo(self): + def test_embedded_document_to_mongo(self): class Person(EmbeddedDocument): name = StringField() age = IntField() @@ -484,14 +484,14 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Employee(name="Bob", age=35, salary=0).to_mongo().keys(), ['_cls', 'name', 'age', 'salary']) - def ztest_embedded_document_to_mongo_id(self): + def test_embedded_document_to_mongo_id(self): class SubDoc(EmbeddedDocument): id = StringField(required=True) sub_doc = SubDoc(id="abc") self.assertEqual(sub_doc.to_mongo().keys(), ['id']) - def ztest_embedded_document(self): + def test_embedded_document(self): """Ensure that embedded documents are set up correctly. """ class Comment(EmbeddedDocument): @@ -500,7 +500,7 @@ class InstanceTest(unittest.TestCase): self.assertTrue('content' in Comment._fields) self.assertFalse('id' in Comment._fields) - def ztest_embedded_document_instance(self): + def test_embedded_document_instance(self): """Ensure that embedded documents can reference parent instance """ class Embedded(EmbeddedDocument): @@ -515,7 +515,7 @@ class InstanceTest(unittest.TestCase): doc = Doc.objects.get() self.assertEqual(doc, doc.embedded_field._instance) - def ztest_embedded_document_complex_instance(self): + def test_embedded_document_complex_instance(self): """Ensure that embedded documents in complex fields can reference parent instance""" class Embedded(EmbeddedDocument): @@ -530,7 +530,7 @@ class InstanceTest(unittest.TestCase): doc = Doc.objects.get() self.assertEqual(doc, doc.embedded_field[0]._instance) - def ztest_instance_is_set_on_setattr(self): + def test_instance_is_set_on_setattr(self): class Email(EmbeddedDocument): email = EmailField() @@ -547,7 +547,7 @@ class InstanceTest(unittest.TestCase): acc1 = Account.objects.first() self.assertTrue(hasattr(acc1._data["email"], "_instance")) - def ztest_document_clean(self): + def test_document_clean(self): class TestDocument(Document): status = StringField() pub_date = DateTimeField() @@ -581,7 +581,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(type(t.pub_date), datetime) - def ztest_document_embedded_clean(self): + def test_document_embedded_clean(self): class TestEmbeddedDocument(EmbeddedDocument): x = IntField(required=True) y = IntField(required=True) @@ -617,7 +617,7 @@ class InstanceTest(unittest.TestCase): t = TestDocument(doc=TestEmbeddedDocument(x=15, y=35, z=5)) t.save(clean=False) - def ztest_save(self): + def test_save(self): """Ensure that a document may be saved in the database. """ # Create person object and save it to the database @@ -642,7 +642,7 @@ class InstanceTest(unittest.TestCase): except ValidationError: self.fail() - def ztest_save_to_a_value_that_equates_to_false(self): + def test_save_to_a_value_that_equates_to_false(self): class Thing(EmbeddedDocument): count = IntField() @@ -662,7 +662,7 @@ class InstanceTest(unittest.TestCase): user.reload() self.assertEqual(user.thing.count, 0) - def ztest_save_max_recursion_not_hit(self): + def test_save_max_recursion_not_hit(self): class Person(Document): name = StringField() @@ -688,7 +688,7 @@ class InstanceTest(unittest.TestCase): p0.name = 'wpjunior' p0.save() - def ztest_save_max_recursion_not_hit_with_file_field(self): + def test_save_max_recursion_not_hit_with_file_field(self): class Foo(Document): name = StringField() @@ -712,7 +712,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(b.picture, b.bar.picture, b.bar.bar.picture) - def ztest_save_cascades(self): + def test_save_cascades(self): class Person(Document): name = StringField() @@ -735,7 +735,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) - def ztest_save_cascade_kwargs(self): + def test_save_cascade_kwargs(self): class Person(Document): name = StringField() @@ -756,7 +756,7 @@ class InstanceTest(unittest.TestCase): p2.reload() self.assertEqual(p1.name, p2.parent.name) - def ztest_save_cascade_meta_false(self): + def test_save_cascade_meta_false(self): class Person(Document): name = StringField() @@ -785,7 +785,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) - def ztest_save_cascade_meta_true(self): + def test_save_cascade_meta_true(self): class Person(Document): name = StringField() @@ -810,7 +810,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertNotEqual(p1.name, p.parent.name) - def ztest_save_cascades_generically(self): + def test_save_cascades_generically(self): class Person(Document): name = StringField() @@ -836,7 +836,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) - def ztest_save_atomicity_condition(self): + def test_save_atomicity_condition(self): class Widget(Document): toggle = BooleanField(default=False) @@ -910,7 +910,7 @@ class InstanceTest(unittest.TestCase): self.assertTrue(w1.toggle) self.assertEqual(w1.count, 3) - def ztest_update(self): + def test_update(self): """Ensure that an existing document is updated instead of be overwritten.""" # Create person object and save it to the database @@ -994,7 +994,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.name, None) self.assertEqual(person.age, None) - def ztest_inserts_if_you_set_the_pk(self): + def test_inserts_if_you_set_the_pk(self): p1 = self.Person(name='p1', id=bson.ObjectId()).save() p2 = self.Person(name='p2') p2.id = bson.ObjectId() @@ -1002,7 +1002,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(2, self.Person.objects.count()) - def ztest_can_save_if_not_included(self): + def test_can_save_if_not_included(self): class EmbeddedDoc(EmbeddedDocument): pass @@ -1051,7 +1051,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(my_doc.string_field, "string") self.assertEqual(my_doc.int_field, 1) - def ztest_document_update(self): + def test_document_update(self): def update_not_saved_raises(): person = self.Person(name='dcrosta') @@ -1080,7 +1080,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidQueryError, update_no_op_raises) - def ztest_update_unique_field(self): + def test_update_unique_field(self): class Doc(Document): name = StringField(unique=True) @@ -1090,7 +1090,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(NotUniqueError, lambda: doc2.update(set__name=doc1.name)) - def ztest_embedded_update(self): + def test_embedded_update(self): """ Test update on `EmbeddedDocumentField` fields """ @@ -1114,7 +1114,7 @@ class InstanceTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") - def ztest_embedded_update_db_field(self): + def test_embedded_update_db_field(self): """ Test update on `EmbeddedDocumentField` fields when db_field is other than default. @@ -1141,7 +1141,7 @@ class InstanceTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") - def ztest_save_only_changed_fields(self): + def test_save_only_changed_fields(self): """Ensure save only sets / unsets changed fields """ @@ -1170,7 +1170,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.age, 21) self.assertEqual(person.active, False) - def ztest_query_count_when_saving(self): + def test_query_count_when_saving(self): """Ensure references don't cause extra fetches when saving""" class Organization(Document): name = StringField() @@ -1263,7 +1263,7 @@ class InstanceTest(unittest.TestCase): sub.save() self.assertEqual(q, 3) - def ztest_set_unset_one_operation(self): + def test_set_unset_one_operation(self): """Ensure that $set and $unset actions are performed in the same operation. """ @@ -1285,7 +1285,7 @@ class InstanceTest(unittest.TestCase): foo.save() self.assertEqual(1, q) - def ztest_save_only_changed_fields_recursive(self): + def test_save_only_changed_fields_recursive(self): """Ensure save only sets / unsets changed fields """ @@ -1327,7 +1327,7 @@ class InstanceTest(unittest.TestCase): person = self.Person.objects.get() self.assertFalse(person.comments_dict['first_post'].published) - def ztest_delete(self): + def test_delete(self): """Ensure that document may be deleted using the delete method. """ person = self.Person(name="Test User", age=30) @@ -1336,7 +1336,7 @@ class InstanceTest(unittest.TestCase): person.delete() self.assertEqual(self.Person.objects.count(), 0) - def ztest_save_custom_id(self): + def test_save_custom_id(self): """Ensure that a document may be saved with a custom _id. """ # Create person object and save it to the database @@ -1348,7 +1348,7 @@ class InstanceTest(unittest.TestCase): person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') - def ztest_save_custom_pk(self): + def test_save_custom_pk(self): """Ensure that a document may be saved with a custom _id using pk alias. """ # Create person object and save it to the database @@ -1360,7 +1360,7 @@ class InstanceTest(unittest.TestCase): person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') - def ztest_save_list(self): + def test_save_list(self): """Ensure that a list field may be properly saved. """ class Comment(EmbeddedDocument): @@ -1387,7 +1387,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - def ztest_list_search_by_embedded(self): + def test_list_search_by_embedded(self): class User(Document): username = StringField(required=True) @@ -1439,7 +1439,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual([p1, p2, p4], list(Page.objects.filter(comments__user=u2))) self.assertEqual([p1, p3], list(Page.objects.filter(comments__user=u3))) - def ztest_save_embedded_document(self): + def test_save_embedded_document(self): """Ensure that a document with an embedded document field may be saved in the database. """ @@ -1463,7 +1463,7 @@ class InstanceTest(unittest.TestCase): # Ensure that the 'details' embedded object saved correctly self.assertEqual(employee_obj['details']['position'], 'Developer') - def ztest_embedded_update_after_save(self): + def test_embedded_update_after_save(self): """ Test update of `EmbeddedDocumentField` attached to a newly saved document. @@ -1486,7 +1486,7 @@ class InstanceTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") - def ztest_updating_an_embedded_document(self): + def test_updating_an_embedded_document(self): """Ensure that a document with an embedded document field may be saved in the database. """ @@ -1521,7 +1521,7 @@ class InstanceTest(unittest.TestCase): promoted_employee.reload() self.assertEqual(promoted_employee.details, None) - def ztest_object_mixins(self): + def test_object_mixins(self): class NameMixin(object): name = StringField() @@ -1536,7 +1536,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(['id', 'name', 'widgets'], sorted(Bar._fields.keys())) - def ztest_mixin_inheritance(self): + def test_mixin_inheritance(self): class BaseMixIn(object): count = IntField() data = StringField() @@ -1560,7 +1560,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(t.data, "test") self.assertEqual(t.count, 12) - def ztest_save_reference(self): + def test_save_reference(self): """Ensure that a document reference field may be saved in the database. """ @@ -1596,7 +1596,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - def ztest_duplicate_db_fields_raise_invalid_document_error(self): + def test_duplicate_db_fields_raise_invalid_document_error(self): """Ensure a InvalidDocumentError is thrown if duplicate fields declare the same db_field""" @@ -1607,7 +1607,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, throw_invalid_document_error) - def ztest_invalid_son(self): + def test_invalid_son(self): """Raise an error if loading invalid data""" class Occurrence(EmbeddedDocument): number = IntField() @@ -1624,7 +1624,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, raise_invalid_document) - def ztest_reverse_delete_rule_cascade_and_nullify(self): + def test_reverse_delete_rule_cascade_and_nullify(self): """Ensure that a referenced document is also deleted upon deletion. """ @@ -1655,7 +1655,7 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(BlogPost.objects.count(), 0) - def ztest_reverse_delete_rule_with_document_inheritance(self): + def test_reverse_delete_rule_with_document_inheritance(self): """Ensure that a referenced document is also deleted upon deletion of a child document. """ @@ -1690,7 +1690,7 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(BlogPost.objects.count(), 0) - def ztest_reverse_delete_rule_cascade_and_nullify_complex_field(self): + def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): """Ensure that a referenced document is also deleted upon deletion for complex fields. """ @@ -1724,7 +1724,7 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(BlogPost.objects.count(), 0) - def ztest_reverse_delete_rule_cascade_triggers_pre_delete_signal(self): + def test_reverse_delete_rule_cascade_triggers_pre_delete_signal(self): ''' ensure the pre_delete signal is triggered upon a cascading deletion setup a blog post with content, an author and editor delete the author which triggers deletion of blogpost via cascade @@ -1760,7 +1760,7 @@ class InstanceTest(unittest.TestCase): editor = Editor.objects(name='Max P.').get() self.assertEqual(editor.review_queue, 0) - def ztest_two_way_reverse_delete_rule(self): + def test_two_way_reverse_delete_rule(self): """Ensure that Bi-Directional relationships work with reverse_delete_rule """ @@ -1793,7 +1793,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Bar.objects.count(), 1) # No effect on the BlogPost self.assertEqual(Bar.objects.get().foo, None) - def ztest_invalid_reverse_delete_rules_raise_errors(self): + def test_invalid_reverse_delete_rules_raise_errors(self): def throw_invalid_document_error(): class Blog(Document): @@ -1810,7 +1810,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, throw_invalid_document_error_embedded) - def ztest_reverse_delete_rule_cascade_recurs(self): + def test_reverse_delete_rule_cascade_recurs(self): """Ensure that a chain of documents is also deleted upon cascaded deletion. """ @@ -1847,7 +1847,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() Comment.drop_collection() - def ztest_reverse_delete_rule_deny(self): + def test_reverse_delete_rule_deny(self): """Ensure that a document cannot be referenced if there are still documents referring to it. """ @@ -1902,7 +1902,7 @@ class InstanceTest(unittest.TestCase): A.drop_collection() B.drop_collection() - def ztest_document_hash(self): + def test_document_hash(self): """Test document in list, dict, set """ class User(Document): @@ -1950,7 +1950,7 @@ class InstanceTest(unittest.TestCase): self.assertTrue(u1 in all_user_set) - def ztest_picklable(self): + def test_picklable(self): pickle_doc = PickleTest(number=1, string="One", lists=['1', '2']) pickle_doc.embedded = PickleEmbedded() @@ -1976,7 +1976,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(pickle_doc.string, "Two") self.assertEqual(pickle_doc.lists, ["1", "2", "3"]) - def ztest_dynamic_document_pickle(self): + def test_dynamic_document_pickle(self): pickle_doc = PickleDynamicTest(name="test", number=1, string="One", lists=['1', '2']) pickle_doc.embedded = PickleDyanmicEmbedded(foo="Bar") @@ -1999,13 +1999,13 @@ class InstanceTest(unittest.TestCase): self.assertEqual(resurrected.embedded._dynamic_fields.keys(), pickle_doc.embedded._dynamic_fields.keys()) - def ztest_picklable_on_signals(self): + def test_picklable_on_signals(self): pickle_doc = PickleSignalsTest(number=1, string="One", lists=['1', '2']) pickle_doc.embedded = PickleEmbedded() pickle_doc.save() pickle_doc.delete() - def ztest_throw_invalid_document_error(self): + def test_throw_invalid_document_error(self): # test handles people trying to upsert def throw_invalid_document_error(): @@ -2014,7 +2014,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, throw_invalid_document_error) - def ztest_mutating_documents(self): + def test_mutating_documents(self): class B(EmbeddedDocument): field1 = StringField(default='field1') @@ -2045,7 +2045,7 @@ class InstanceTest(unittest.TestCase): a.reload() self.assertEqual(a.b.field2.c_field, 'new value') - def ztest_can_save_false_values(self): + def test_can_save_false_values(self): """Ensures you can save False values on save""" class Doc(Document): foo = StringField() @@ -2059,7 +2059,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Doc.objects(archived=False).count(), 1) - def ztest_can_save_false_values_dynamic(self): + def test_can_save_false_values_dynamic(self): """Ensures you can save False values on dynamic docs""" class Doc(DynamicDocument): foo = StringField() @@ -2072,7 +2072,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Doc.objects(archived=False).count(), 1) - def ztest_do_not_save_unchanged_references(self): + def test_do_not_save_unchanged_references(self): """Ensures cascading saves dont auto update""" class Job(Document): name = StringField() @@ -2103,7 +2103,7 @@ class InstanceTest(unittest.TestCase): finally: Collection.update = orig_update - def ztest_db_alias_tests(self): + def test_db_alias_tests(self): """ DB Alias tests """ # mongoenginetest - Is default connection alias from setUp() # Register Aliases @@ -2159,7 +2159,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Book._get_collection(), get_db("testdb-2")[Book._get_collection_name()]) self.assertEqual(AuthorBooks._get_collection(), get_db("testdb-3")[AuthorBooks._get_collection_name()]) - def ztest_db_alias_overrides(self): + def test_db_alias_overrides(self): """db_alias can be overriden """ # Register a connection with db_alias testdb-2 @@ -2184,7 +2184,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual('mongoenginetest2', B._get_collection().database.name) - def ztest_db_alias_propagates(self): + def test_db_alias_propagates(self): """db_alias propagates? """ register_connection('testdb-1', 'mongoenginetest2') @@ -2198,7 +2198,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual('testdb-1', B._meta.get('db_alias')) - def ztest_db_ref_usage(self): + def test_db_ref_usage(self): """ DB Ref usage in dict_fields""" class User(Document): @@ -2276,7 +2276,7 @@ class InstanceTest(unittest.TestCase): })]), "1,2") - def ztest_switch_db_instance(self): + def test_switch_db_instance(self): register_connection('testdb-1', 'mongoenginetest2') class Group(Document): @@ -2326,7 +2326,7 @@ class InstanceTest(unittest.TestCase): group = Group.objects.first() self.assertEqual("hello - default", group.name) - def ztest_no_overwritting_no_data_loss(self): + def test_no_overwritting_no_data_loss(self): class User(Document): username = StringField(primary_key=True) @@ -2350,7 +2350,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual("Bar", user._data["foo"]) self.assertEqual([1, 2, 3], user._data["data"]) - def ztest_spaces_in_keys(self): + def test_spaces_in_keys(self): class Embedded(DynamicEmbeddedDocument): pass @@ -2366,7 +2366,7 @@ class InstanceTest(unittest.TestCase): one = Doc.objects.filter(**{'hello world': 1}).count() self.assertEqual(1, one) - def ztest_shard_key(self): + def test_shard_key(self): class LogEntry(Document): machine = StringField() log = StringField() @@ -2391,7 +2391,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) - def ztest_shard_key_primary(self): + def test_shard_key_primary(self): class LogEntry(Document): machine = StringField(primary_key=True) log = StringField() @@ -2416,7 +2416,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) - def ztest_kwargs_simple(self): + def test_kwargs_simple(self): class Embedded(EmbeddedDocument): name = StringField() @@ -2432,7 +2432,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(classic_doc, dict_doc) self.assertEqual(classic_doc._data, dict_doc._data) - def ztest_kwargs_complex(self): + def test_kwargs_complex(self): class Embedded(EmbeddedDocument): name = StringField() @@ -2451,21 +2451,21 @@ class InstanceTest(unittest.TestCase): self.assertEqual(classic_doc, dict_doc) self.assertEqual(classic_doc._data, dict_doc._data) - def ztest_positional_creation(self): + def test_positional_creation(self): """Ensure that document may be created using positional arguments. """ person = self.Person("Test User", 42) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - def ztest_mixed_creation(self): + def test_mixed_creation(self): """Ensure that document may be created using mixed arguments. """ person = self.Person("Test User", age=42) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - def ztest_mixed_creation_dynamic(self): + def test_mixed_creation_dynamic(self): """Ensure that document may be created using mixed arguments. """ class Person(DynamicDocument): @@ -2475,7 +2475,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - def ztest_bad_mixed_creation(self): + def test_bad_mixed_creation(self): """Ensure that document gives correct error when duplicating arguments """ def construct_bad_instance(): @@ -2483,7 +2483,7 @@ class InstanceTest(unittest.TestCase): self.assertRaises(TypeError, construct_bad_instance) - def ztest_data_contains_id_field(self): + def test_data_contains_id_field(self): """Ensure that asking for _data returns 'id' """ class Person(Document): @@ -2496,7 +2496,7 @@ class InstanceTest(unittest.TestCase): self.assertTrue('id' in person._data.keys()) self.assertEqual(person._data.get('id'), person.id) - def ztest_complex_nesting_document_and_embedded_document(self): + def test_complex_nesting_document_and_embedded_document(self): class Macro(EmbeddedDocument): value = DynamicField(default="UNDEFINED") @@ -2537,7 +2537,7 @@ class InstanceTest(unittest.TestCase): system = NodesSystem.objects.first() self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value) - def ztest_embedded_document_equality(self): + def test_embedded_document_equality(self): class Test(Document): field = StringField(required=True) From 89b9b60e0c218654091b34e6139f1c44f1368b08 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 11:27:10 +0100 Subject: [PATCH 85/89] Geo SON tweaks --- mongoengine/queryset/transform.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 8e88e9fe..d72d97a3 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -117,19 +117,24 @@ def query(_doc_cls=None, _field_operation=False, **query): mongo_query[key].update(value) # $maxDistance needs to come last - convert to SON value_dict = mongo_query[key] - if ('$maxDistance' in value_dict and '$near' in value_dict and - isinstance(value_dict['$near'], dict)): - + if ('$maxDistance' in value_dict and '$near' in value_dict): value_son = SON() - for k, v in value_dict.iteritems(): - if k == '$maxDistance': - continue - value_son[k] = v - if (get_connection().max_wire_version <= 1): - value_son['$maxDistance'] = value_dict['$maxDistance'] + if isinstance(value_dict['$near'], dict): + for k, v in value_dict.iteritems(): + if k == '$maxDistance': + continue + value_son[k] = v + if (get_connection().max_wire_version <= 1): + value_son['$maxDistance'] = value_dict['$maxDistance'] + else: + value_son['$near'] = SON(value_son['$near']) + value_son['$near']['$maxDistance'] = value_dict['$maxDistance'] else: - value_son['$near'] = SON(value_son['$near']) - value_son['$near']['$maxDistance'] = value_dict['$maxDistance'] + for k, v in value_dict.iteritems(): + if k == '$maxDistance': + continue + value_son[k] = v + value_son['$maxDistance'] = value_dict['$maxDistance'] mongo_query[key] = value_son else: From 1502dda2aba65aeff33d946a75b5bca06076c192 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 11:33:56 +0100 Subject: [PATCH 86/89] Fixed ReferenceField inside nested ListFields dereferencing problem #368 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 13cd1b69..5b8eb623 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Fixed ReferenceField inside nested ListFields dereferencing problem #368 - Added the ability to reload specific document fields #100 - Added db_alias support and fixes for custom map/reduce output #586 - post_save signal now has access to delta information about field changes #594 #589 From 1743ab7812573e376e489d4c7036909b8278e133 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 11:38:06 +0100 Subject: [PATCH 87/89] Changelog update #567 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5b8eb623..38d46828 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Implemented equality between Documents and DBRefs #597 - Fixed ReferenceField inside nested ListFields dereferencing problem #368 - Added the ability to reload specific document fields #100 - Added db_alias support and fixes for custom map/reduce output #586 From 47ebac0276c49bfe825e7045d511ad79b41d350c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 11:59:35 +0100 Subject: [PATCH 88/89] Add authentication_source option to register_connection #178 #464 #573 #580 #590 --- docs/changelog.rst | 1 + tests/test_connection.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 38d46828..c4f9ec23 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Add authentication_source option to register_connection #178 #464 #573 #580 #590 - Implemented equality between Documents and DBRefs #597 - Fixed ReferenceField inside nested ListFields dereferencing problem #368 - Added the ability to reload specific document fields #100 diff --git a/tests/test_connection.py b/tests/test_connection.py index a5b1b089..6cdbd654 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -48,7 +48,7 @@ class ConnectionTest(unittest.TestCase): connect('mongoenginetest', alias='testdb2') actual_connection = get_connection('testdb2') - self.assertIs(expected_connection, actual_connection) + self.assertEqual(expected_connection, actual_connection) def test_connect_uri(self): """Ensure that the connect() method works properly with uri's From 4d7492f6826a935813a4cc366af23079cb18e0ab Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jun 2014 12:10:17 +0100 Subject: [PATCH 89/89] Changelog & Author updates #425 #507 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index c6c47d79..378f2234 100644 --- a/AUTHORS +++ b/AUTHORS @@ -200,3 +200,4 @@ that much better: * Andrei Zbikowski (https://github.com/b1naryth1ef) * Ronald van Rij (https://github.com/ronaldvanrij) * François Schmidts (https://github.com/jaesivsm) + * Eric Plumb (https://github.com/professorplumb) diff --git a/docs/changelog.rst b/docs/changelog.rst index c4f9ec23..27bcd46f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== +- Dynamic fields in embedded documents now visible to queryset.only() / qs.exclude() #425 #507 - Add authentication_source option to register_connection #178 #464 #573 #580 #590 - Implemented equality between Documents and DBRefs #597 - Fixed ReferenceField inside nested ListFields dereferencing problem #368