From 0d4afad342b6412d322c3ef20c61c7b51e9698ef Mon Sep 17 00:00:00 2001
From: Andrei Zbikowski <andrei@spoton.com>
Date: Fri, 24 Jan 2014 16:54:29 -0600
Subject: [PATCH 01/73] Fixes issue with recursive embedded document errors

---
 mongoengine/base/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index f5eae8ff..c625f8b8 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -317,7 +317,7 @@ class BaseDocument(object):
             pk = "None"
             if hasattr(self, 'pk'):
                 pk = self.pk
-            elif self._instance:
+            elif self._instance and hasattr(self._instance, 'pk'):
                 pk = self._instance.pk
             message = "ValidationError (%s:%s) " % (self._class_name, pk)
             raise ValidationError(message, errors=errors)

From b085993901e57797bdf8944e86a8a2611ac1e823 Mon Sep 17 00:00:00 2001
From: "Brian J. Dowling" <bjd-dev@simplicity.net>
Date: Mon, 27 Jan 2014 23:05:29 +0000
Subject: [PATCH 02/73] Allow dynamic dictionary-style field access

Allows the doc[key] syntax to work for dynamicembeddeddocument fields

Fixes #559
---
 mongoengine/base/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index f5eae8ff..c5ef087c 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -182,7 +182,7 @@ class BaseDocument(object):
         """Dictionary-style field access, set a field's value.
         """
         # Ensure that the field exists before settings its value
-        if name not in self._fields:
+        if not self._dynamic and name not in self._fields:
             raise KeyError(name)
         return setattr(self, name, value)
 

From 9b2080d036859c4456fc56c3085a14e791853035 Mon Sep 17 00:00:00 2001
From: "Brian J. Dowling" <bjd-dev@simplicity.net>
Date: Tue, 28 Jan 2014 22:10:26 -0500
Subject: [PATCH 03/73] Added a test for allowing dynamic dictionary-style
 field access

Closes #559
---
 tests/document/dynamic.py | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py
index 6263e68c..bf69cb27 100644
--- a/tests/document/dynamic.py
+++ b/tests/document/dynamic.py
@@ -292,6 +292,44 @@ class DynamicTest(unittest.TestCase):
         person.save()
         self.assertEqual(Person.objects.first().age, 35)
 
+    def test_dynamic_and_embedded_dict_access(self):
+        """Ensure embedded dynamic documents work with dict[] style access"""
+
+        class Address(EmbeddedDocument):
+            city = StringField()
+
+        class Person(DynamicDocument):
+            name = StringField()
+
+        Person.drop_collection()
+
+        Person(name="Ross", address=Address(city="London")).save()
+
+        person = Person.objects.first()
+        person.attrval = "This works"
+
+        person["phone"] = "555-1212" # but this should too
+
+        # Same thing two levels deep
+        person["address"]["city"] = "Lundenne"
+        person.save()
+
+        self.assertEqual(Person.objects.first().address.city, "Lundenne")
+
+        self.assertEqual(Person.objects.first().phone, "555-1212")
+
+        person = Person.objects.first()
+        person.address = Address(city="Londinium")
+        person.save()
+
+        self.assertEqual(Person.objects.first().address.city, "Londinium")
+
+
+        person = Person.objects.first()
+        person["age"] = 35
+        person.save()
+        self.assertEqual(Person.objects.first().age, 35)
+
 
 if __name__ == '__main__':
     unittest.main()

From db1e69813bc0b31957e7589723a2f80cda74454b Mon Sep 17 00:00:00 2001
From: Frank Battaglia <frank@Franks-MacBook-Pro.local>
Date: Thu, 31 Oct 2013 20:43:07 -0400
Subject: [PATCH 04/73] add atomic conditions to save

Conflicts:
	mongoengine/document.py
---
 mongoengine/document.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index 114778eb..f870a078 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -13,7 +13,8 @@ from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass,
                               BaseDocument, BaseDict, BaseList,
                               ALLOW_INHERITANCE, get_document)
 from mongoengine.errors import ValidationError
-from mongoengine.queryset import OperationError, NotUniqueError, QuerySet
+from mongoengine.queryset import (OperationError, NotUniqueError,
+                                  QuerySet, transform)
 from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME
 from mongoengine.context_managers import switch_db, switch_collection
 
@@ -180,7 +181,7 @@ class Document(BaseDocument):
 
     def save(self, force_insert=False, validate=True, clean=True,
              write_concern=None,  cascade=None, cascade_kwargs=None,
-             _refs=None, **kwargs):
+             _refs=None, save_condition=None, **kwargs):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
         created.
@@ -203,7 +204,8 @@ class Document(BaseDocument):
         :param cascade_kwargs: (optional) kwargs dictionary to be passed throw
             to cascading saves.  Implies ``cascade=True``.
         :param _refs: A list of processed references used in cascading saves
-
+        :param save_condition: only perform save if matching record in db
+            satisfies condition(s) (e.g., version number)
         .. versionchanged:: 0.5
             In existing documents it only saves changed fields using
             set / unset.  Saves are cascaded and any
@@ -217,6 +219,9 @@ class Document(BaseDocument):
             meta['cascade'] = True.  Also you can pass different kwargs to
             the cascade save using cascade_kwargs which overwrites the
             existing kwargs with custom values.
+        .. versionchanged:: 0.8.5
+            Optional save_condition that only overwrites existing documents
+            if the condition is satisfied in the current db record.
         """
         signals.pre_save.send(self.__class__, document=self)
 
@@ -230,7 +235,8 @@ class Document(BaseDocument):
 
         created = ('_id' not in doc or self._created or force_insert)
 
-        signals.pre_save_post_validation.send(self.__class__, document=self, created=created)
+        signals.pre_save_post_validation.send(self.__class__, document=self,
+                                              created=created)
 
         try:
             collection = self._get_collection()
@@ -243,7 +249,12 @@ class Document(BaseDocument):
                 object_id = doc['_id']
                 updates, removals = self._delta()
                 # Need to add shard key to query, or you get an error
-                select_dict = {'_id': object_id}
+                if save_condition is not None:
+                    select_dict = transform.query(self.__class__,
+                                                  **save_condition)
+                else:
+                    select_dict = {}
+                select_dict['_id'] = object_id
                 shard_key = self.__class__._meta.get('shard_key', tuple())
                 for k in shard_key:
                     actual_key = self._db_field_map.get(k, k)
@@ -263,10 +274,12 @@ class Document(BaseDocument):
                 if removals:
                     update_query["$unset"] = removals
                 if updates or removals:
+                    upsert = save_condition is None
                     last_error = collection.update(select_dict, update_query,
-                                                   upsert=True, **write_concern)
+                                                   upsert=upsert, **write_concern)
                     created = is_new_object(last_error)
 
+
             if cascade is None:
                 cascade = self._meta.get('cascade', False) or cascade_kwargs is not None
 

From 673a966541e8b2083616497378f54c021f5b0c08 Mon Sep 17 00:00:00 2001
From: Frank Battaglia <frank@Franks-MacBook-Pro.local>
Date: Tue, 12 Nov 2013 15:22:27 -0500
Subject: [PATCH 05/73] add tests for save_condition kwarg in document.save()

---
 tests/document/instance.py | 62 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index 07db85a0..4747c0ed 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -820,6 +820,68 @@ class InstanceTest(unittest.TestCase):
         p1.reload()
         self.assertEqual(p1.name, p.parent.name)
 
+    def test_save_atomicity_condition(self):
+
+        class Widget(Document):
+            toggle = BooleanField(default=False)
+            count = IntField(default=0)
+            save_id = UUIDField()
+
+        def flip(widget):
+            widget.toggle = not widget.toggle
+            widget.count += 1
+
+        def UUID(i):
+            return uuid.UUID(int=i)
+
+        Widget.drop_collection()
+        
+        w1 = Widget(toggle=False, save_id=UUID(1))
+
+        # ignore save_condition on new record creation
+        w1.save(save_condition={'save_id':UUID(42)})
+        w1.reload()
+        self.assertFalse(w1.toggle)
+        self.assertEqual(w1.save_id, UUID(1))
+        self.assertEqual(w1.count, 0)
+
+        # mismatch in save_condition prevents save
+        flip(w1)
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 1)
+        w1.save(save_condition={'save_id':UUID(42)})
+        w1.reload()
+        self.assertFalse(w1.toggle)
+        self.assertEqual(w1.count, 0)
+
+        # matched save_condition allows save
+        flip(w1)
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 1)
+        w1.save(save_condition={'save_id':UUID(1)})
+        w1.reload()
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 1)
+
+        # save_condition can be used to ensure atomic read & updates
+        # i.e., prevent interleaved reads and writes from separate contexts
+        w2 = Widget.objects.get()
+        self.assertEqual(w1, w2)
+        old_id = w1.save_id
+
+        flip(w1)
+        w1.save(save_condition={'save_id':old_id})
+        w1.reload()
+        self.assertFalse(w1.toggle)
+        self.assertEqual(w1.count, 2)
+        flip(w2)
+        flip(w2)
+        w2.save(save_condition={'save_id':old_id})
+        w2.reload()
+        self.assertFalse(w1.toggle)
+        self.assertEqual(w1.count, 2)
+        
+
     def test_update(self):
         """Ensure that an existing document is updated instead of be
         overwritten."""

From 0a2dbbc58b5ae0df92e5465c1a43d46b71d0adb4 Mon Sep 17 00:00:00 2001
From: Frank Battaglia <frank@Franks-MacBook-Pro.local>
Date: Tue, 12 Nov 2013 16:01:56 -0500
Subject: [PATCH 06/73] add tests for mongo query operators

---
 tests/document/instance.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index 4747c0ed..df6b4fcb 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -870,6 +870,7 @@ class InstanceTest(unittest.TestCase):
         old_id = w1.save_id
 
         flip(w1)
+        w1.save_id = UUID(2)
         w1.save(save_condition={'save_id':old_id})
         w1.reload()
         self.assertFalse(w1.toggle)
@@ -878,8 +879,20 @@ class InstanceTest(unittest.TestCase):
         flip(w2)
         w2.save(save_condition={'save_id':old_id})
         w2.reload()
-        self.assertFalse(w1.toggle)
-        self.assertEqual(w1.count, 2)
+        self.assertFalse(w2.toggle)
+        self.assertEqual(w2.count, 2)
+
+        # save_condition uses mongoengine-style operator syntax
+        flip(w1)
+        w1.save(save_condition={'count__lt':w1.count})
+        w1.reload()
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 3)
+        flip(w1)
+        w1.save(save_condition={'count__gte':w1.count})
+        w1.reload()
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 3)
         
 
     def test_update(self):

From 86363986fc0521262676811835469efdb1530ddb Mon Sep 17 00:00:00 2001
From: Frank Battaglia <frank@Franks-MacBook-Pro.local>
Date: Tue, 12 Nov 2013 16:37:20 -0500
Subject: [PATCH 07/73] whitespace

---
 tests/document/instance.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index df6b4fcb..acb26c6d 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -894,7 +894,6 @@ class InstanceTest(unittest.TestCase):
         self.assertTrue(w1.toggle)
         self.assertEqual(w1.count, 3)
         
-
     def test_update(self):
         """Ensure that an existing document is updated instead of be
         overwritten."""

From 9d125c9e797643fbe776114407da321560ef7ca2 Mon Sep 17 00:00:00 2001
From: Frank Battaglia <frank@franks-macbook-pro-wired.home>
Date: Sun, 23 Feb 2014 19:37:42 -0500
Subject: [PATCH 08/73] inherit parent Document type _auto_id_field value

---
 mongoengine/base/metaclasses.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py
index ff5afddf..4b2e8b9b 100644
--- a/mongoengine/base/metaclasses.py
+++ b/mongoengine/base/metaclasses.py
@@ -359,7 +359,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
                     new_class.id = field
 
         # Set primary key if not defined by the document
-        new_class._auto_id_field = False
+        new_class._auto_id_field = getattr(parent_doc_cls,
+                                           '_auto_id_field', False)
         if not new_class._meta.get('id_field'):
             new_class._auto_id_field = True
             new_class._meta['id_field'] = 'id'

From ef55e6d476a89838103f70e420b6535d8186ac99 Mon Sep 17 00:00:00 2001
From: Vlad Zloteanu <vlad.zloteanu@dolead.com>
Date: Sat, 1 Mar 2014 17:51:59 +0100
Subject: [PATCH 09/73] fixes MongoEngine/mongoengine#589

---
 mongoengine/document.py | 2 +-
 tests/test_signals.py   | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index 114778eb..3aaede27 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -296,9 +296,9 @@ class Document(BaseDocument):
         if id_field not in self._meta.get('shard_key', []):
             self[id_field] = self._fields[id_field].to_python(object_id)
 
+        signals.post_save.send(self.__class__, document=self, created=created)
         self._clear_changed_fields()
         self._created = False
-        signals.post_save.send(self.__class__, document=self, created=created)
         return self
 
     def cascade_save(self, *args, **kwargs):
diff --git a/tests/test_signals.py b/tests/test_signals.py
index 50e5e6b8..3d0cbb3e 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -54,7 +54,9 @@ class SignalTests(unittest.TestCase):
 
             @classmethod
             def post_save(cls, sender, document, **kwargs):
+                dirty_keys = document._delta()[0].keys() + document._delta()[1].keys()
                 signal_output.append('post_save signal, %s' % document)
+                signal_output.append('post_save dirty keys, %s' % dirty_keys)
                 if 'created' in kwargs:
                     if kwargs['created']:
                         signal_output.append('Is created')
@@ -203,6 +205,7 @@ class SignalTests(unittest.TestCase):
             "pre_save_post_validation signal, Bill Shakespeare",
             "Is created",
             "post_save signal, Bill Shakespeare",
+            "post_save dirty keys, ['name']",
             "Is created"
         ])
 
@@ -213,6 +216,7 @@ class SignalTests(unittest.TestCase):
             "pre_save_post_validation signal, William Shakespeare",
             "Is updated",
             "post_save signal, William Shakespeare",
+            "post_save dirty keys, ['name']",
             "Is updated"
         ])
 

From 19314e7e06c3b4b393b0017868182c689c75f9f9 Mon Sep 17 00:00:00 2001
From: Kirill Pavlov <kp@multichannel.net>
Date: Mon, 3 Mar 2014 13:09:26 +0800
Subject: [PATCH 10/73] fix docstring for DictField

---
 mongoengine/fields.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 82642cda..b0e51a24 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -760,7 +760,7 @@ class DictField(ComplexBaseField):
     similar to an embedded document, but the structure is not defined.
 
     .. note::
-        Required means it cannot be empty - as the default for ListFields is []
+        Required means it cannot be empty - as the default for DictFields is {}
 
     .. versionadded:: 0.3
     .. versionchanged:: 0.5 - Can now handle complex / varying types of data

From c82f4f0d45e6e70bf8410d28415602ddea6794e3 Mon Sep 17 00:00:00 2001
From: Phil Freo <phil@philfreo.com>
Date: Fri, 7 Mar 2014 13:29:29 -0800
Subject: [PATCH 11/73] clarifying the 'push' atomic update docs

the first time I read this I was all like... "no duh it will remove either the first or the last, but which does it do???"
---
 docs/guide/querying.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst
index 32cbb94e..7168b5e8 100644
--- a/docs/guide/querying.rst
+++ b/docs/guide/querying.rst
@@ -499,11 +499,13 @@ that you may use with these methods:
 * ``dec`` -- decrement a value by a given amount
 * ``push`` -- append a value to a list
 * ``push_all`` -- append several values to a list
-* ``pop`` -- remove the first or last element of a list
+* ``pop`` -- remove the first or last element of a list `depending on the value`_
 * ``pull`` -- remove a value from a list
 * ``pull_all`` -- remove several values from a list
 * ``add_to_set`` -- add value to a list only if its not in the list already
 
+.. _depending on the value: http://docs.mongodb.org/manual/reference/operator/update/pop/
+
 The syntax for atomic updates is similar to the querying syntax, but the
 modifier comes before the field, not after it::
 

From d27a1103fac41fd3eb4231a89fc35044641b317a Mon Sep 17 00:00:00 2001
From: Damien Churchill <damoxc@gmail.com>
Date: Wed, 12 Mar 2014 17:19:49 +0000
Subject: [PATCH 12/73] workaround a dateutil bug

In the latest released version of dateutil, there's a bug whereby a TypeError
can be raised whilst parsing a date. This is because it calls a method which
it expects to return 2 arguments, however it can return 1 depending upon the
input, which results in a TypeError: ArgType not iterable exception. Since
this is equivalent to a failed parse anyway, we can treat it the same as a
ValueError.
---
 mongoengine/fields.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 82642cda..85a1a743 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -391,7 +391,7 @@ class DateTimeField(BaseField):
         if dateutil:
             try:
                 return dateutil.parser.parse(value)
-            except ValueError:
+            except (TypeError, ValueError):
                 return None
 
         # split usecs, because they are not recognized by strptime.

From 5d9ec0b20854eac476f974b495a1d444c7134bf8 Mon Sep 17 00:00:00 2001
From: Nicolas Despres <nicolas.despres@gmail.com>
Date: Mon, 17 Mar 2014 17:19:17 +0100
Subject: [PATCH 13/73] Save is called on the document not the file field.

---
 docs/guide/gridfs.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst
index 596585de..68e7a6d2 100644
--- a/docs/guide/gridfs.rst
+++ b/docs/guide/gridfs.rst
@@ -46,7 +46,7 @@ slightly different manner.  First, a new file must be created by calling the
     marmot.photo.write('some_more_image_data')
     marmot.photo.close()
 
-    marmot.photo.save()
+    marmot.save()
 
 Deletion
 --------

From c1f88a4e14f319b8a849dad44849d6f6884c8e75 Mon Sep 17 00:00:00 2001
From: Falcon Dai <me@falcondai.com>
Date: Mon, 17 Mar 2014 22:29:53 -0500
Subject: [PATCH 14/73] minor change to geo-related docs

---
 docs/guide/defining-documents.rst |  2 ++
 mongoengine/fields.py             | 13 +++++++++----
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst
index 5d8b628a..07bce3bb 100644
--- a/docs/guide/defining-documents.rst
+++ b/docs/guide/defining-documents.rst
@@ -531,6 +531,8 @@ field name to the index definition.
 Sometimes its more efficient to index parts of Embedded / dictionary fields,
 in this case use 'dot' notation to identify the value to index eg: `rank.title`
 
+.. _geospatial-indexes:
+
 Geospatial indexes
 ------------------
 
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 82642cda..48db6dfd 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -1613,7 +1613,12 @@ class UUIDField(BaseField):
 
 
 class GeoPointField(BaseField):
-    """A list storing a latitude and longitude.
+    """A list storing a longitude and latitude coordinate. 
+
+    .. note:: this represents a generic point in a 2D plane and a legacy way of 
+        representing a geo point. It admits 2d indexes but not "2dsphere" indexes 
+        in MongoDB > 2.4 which are more natural for modeling geospatial points. 
+        See :ref:`geospatial-indexes` 
 
     .. versionadded:: 0.4
     """
@@ -1635,7 +1640,7 @@ class GeoPointField(BaseField):
 
 
 class PointField(GeoJsonBaseField):
-    """A geo json field storing a latitude and longitude.
+    """A GeoJSON field storing a longitude and latitude coordinate.
 
     The data is represented as:
 
@@ -1654,7 +1659,7 @@ class PointField(GeoJsonBaseField):
 
 
 class LineStringField(GeoJsonBaseField):
-    """A geo json field storing a line of latitude and longitude coordinates.
+    """A GeoJSON field storing a line of longitude and latitude coordinates.
 
     The data is represented as:
 
@@ -1672,7 +1677,7 @@ class LineStringField(GeoJsonBaseField):
 
 
 class PolygonField(GeoJsonBaseField):
-    """A geo json field storing a polygon of latitude and longitude coordinates.
+    """A GeoJSON field storing a polygon of longitude and latitude coordinates.
 
     The data is represented as:
 

From 4d7b988018c2d150401fb96db36da1ebdd6a2c4d Mon Sep 17 00:00:00 2001
From: poly <mcx_221@foxmail.com>
Date: Tue, 1 Apr 2014 19:52:21 +0800
Subject: [PATCH 15/73] Fixed uncorrectly split a query key, when it ends with
 "_"

---
 mongoengine/queryset/transform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index e31a8b7d..27e41ad2 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -38,7 +38,7 @@ def query(_doc_cls=None, _field_operation=False, **query):
             mongo_query.update(value)
             continue
 
-        parts = key.split('__')
+        parts = key.rsplit('__')
         indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()]
         parts = [part for part in parts if not part.isdigit()]
         # Check for an operator and transform to mongo-style if there is

From 803caddbd40ab2ea6cbee8e6b65a5921277fcafc Mon Sep 17 00:00:00 2001
From: Dmitry Konishchev <konishchev@gmail.com>
Date: Wed, 9 Apr 2014 14:25:53 +0400
Subject: [PATCH 16/73] Raise NotUniqueError in Document.update() on
 pymongo.errors.DuplicateKeyError

---
 mongoengine/queryset/base.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index c2ad027e..c34c1479 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -443,6 +443,8 @@ class BaseQuerySet(object):
                 return result
             elif result:
                 return result['n']
+        except pymongo.errors.DuplicateKeyError, err:
+            raise NotUniqueError(u'Update failed (%s)' % unicode(err))
         except pymongo.errors.OperationFailure, err:
             if unicode(err) == u'multi not coded yet':
                 message = u'update() method requires MongoDB 1.1.3+'

From b45a601ad2a208b209af5e8f45893aaf16ebd59f Mon Sep 17 00:00:00 2001
From: Dmitry Konishchev <konishchev@gmail.com>
Date: Tue, 15 Apr 2014 19:32:42 +0400
Subject: [PATCH 17/73] Test raising NotUniqueError by Document.update()

---
 tests/document/instance.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index 07db85a0..c57102da 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -15,7 +15,7 @@ from tests.fixtures import (PickleEmbedded, PickleTest, PickleSignalsTest,
 
 from mongoengine import *
 from mongoengine.errors import (NotRegistered, InvalidDocumentError,
-                                InvalidQueryError)
+                                InvalidQueryError, NotUniqueError)
 from mongoengine.queryset import NULLIFY, Q
 from mongoengine.connection import get_db
 from mongoengine.base import get_document
@@ -990,6 +990,16 @@ class InstanceTest(unittest.TestCase):
 
         self.assertRaises(InvalidQueryError, update_no_op_raises)
 
+    def test_update_unique_field(self):
+        class Doc(Document):
+            name = StringField(unique=True)
+
+        doc1 = Doc(name="first").save()
+        doc2 = Doc(name="second").save()
+
+        self.assertRaises(NotUniqueError, lambda:
+                          doc2.update(set__name=doc1.name))
+
     def test_embedded_update(self):
         """
         Test update on `EmbeddedDocumentField` fields

From 12809ebc741dc04677a4e3b0f7c3ed5c23563d7f Mon Sep 17 00:00:00 2001
From: Jatin Chopra <jatin@users.noreply.github.com>
Date: Tue, 6 May 2014 00:25:55 -0700
Subject: [PATCH 18/73] Updated Jatin's name and github name

---
 AUTHORS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index d6994d50..5fa0cb3d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -171,7 +171,7 @@ that much better:
  * Michael Bartnett (https://github.com/michaelbartnett)
  * Alon Horev (https://github.com/alonho)
  * Kelvin Hammond (https://github.com/kelvinhammond)
- * Jatin- (https://github.com/jatin-)
+ * Jatin Chopra (https://github.com/jatin)
  * Paul Uithol (https://github.com/PaulUithol)
  * Thom Knowles (https://github.com/fleat)
  * Paul (https://github.com/squamous)

From babbc8bcd64893a328f34d1cde17dd7125b6dd2f Mon Sep 17 00:00:00 2001
From: Ronald van Rij <ronald@caldrono.com>
Date: Mon, 5 May 2014 16:39:55 +0200
Subject: [PATCH 19/73] When using autogenerated document ids in a sharded
 collection, do set that id back into the Document

---
 mongoengine/document.py    | 2 +-
 tests/document/instance.py | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index 114778eb..e09503c8 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -293,7 +293,7 @@ class Document(BaseDocument):
                 raise NotUniqueError(message % unicode(err))
             raise OperationError(message % unicode(err))
         id_field = self._meta['id_field']
-        if id_field not in self._meta.get('shard_key', []):
+        if created or id_field not in self._meta.get('shard_key', []):
             self[id_field] = self._fields[id_field].to_python(object_id)
 
         self._clear_changed_fields()
diff --git a/tests/document/instance.py b/tests/document/instance.py
index 07db85a0..14cc0074 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -2281,6 +2281,8 @@ class InstanceTest(unittest.TestCase):
         log.machine = "Localhost"
         log.save()
 
+        self.assertIsNotNone(log.id)
+
         log.log = "Saving"
         log.save()
 
@@ -2304,6 +2306,8 @@ class InstanceTest(unittest.TestCase):
         log.machine = "Localhost"
         log.save()
 
+        self.assertIsNotNone(log.id)
+
         log.log = "Saving"
         log.save()
 

From 9544b7d9689bb2ef3672aa230b3829400be16414 Mon Sep 17 00:00:00 2001
From: Ronald van Rij <ronald@caldrono.com>
Date: Fri, 9 May 2014 14:33:18 +0200
Subject: [PATCH 20/73] Fixed unit test which used assertIsNotNone

---
 tests/document/instance.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index 14cc0074..7db452d9 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -2281,7 +2281,7 @@ class InstanceTest(unittest.TestCase):
         log.machine = "Localhost"
         log.save()
 
-        self.assertIsNotNone(log.id)
+        self.assertTrue(log.id is not None)
 
         log.log = "Saving"
         log.save()
@@ -2306,7 +2306,7 @@ class InstanceTest(unittest.TestCase):
         log.machine = "Localhost"
         log.save()
 
-        self.assertIsNotNone(log.id)
+        self.assertTrue(log.id is not None)
 
         log.log = "Saving"
         log.save()

From abcacc82f359d37922dc6c19c6b860b4c633e8e0 Mon Sep 17 00:00:00 2001
From: Stefan Wojcik <wojcikstefan@gmail.com>
Date: Wed, 21 May 2014 22:21:46 -0700
Subject: [PATCH 21/73] dont use a system collection

---
 tests/document/instance.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index 07db85a0..6c4da162 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -2411,7 +2411,7 @@ class InstanceTest(unittest.TestCase):
                 for parameter_name, parameter in self.parameters.iteritems():
                     parameter.expand()
 
-        class System(Document):
+        class NodesSystem(Document):
             name = StringField(required=True)
             nodes = MapField(ReferenceField(Node, dbref=False))
 
@@ -2419,18 +2419,18 @@ class InstanceTest(unittest.TestCase):
                 for node_name, node in self.nodes.iteritems():
                     node.expand()
                     node.save(*args, **kwargs)
-                super(System, self).save(*args, **kwargs)
+                super(NodesSystem, self).save(*args, **kwargs)
 
-        System.drop_collection()
+        NodesSystem.drop_collection()
         Node.drop_collection()
 
-        system = System(name="system")
+        system = NodesSystem(name="system")
         system.nodes["node"] = Node()
         system.save()
         system.nodes["node"].parameters["param"] = Parameter()
         system.save()
 
-        system = System.objects.first()
+        system = NodesSystem.objects.first()
         self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value)
 
     def test_embedded_document_equality(self):

From 3faf3c84be55f709d828ea7837db988c56b0239e Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Tue, 27 May 2014 16:33:38 -0300
Subject: [PATCH 22/73] Avoid to open all documents from cursors in an if stmt

Using a cursos in an if statement:

cursor = Collection.objects

	if cursor:
		(...)

Will open all documents, because there are not an __nonzero__ method.
This change check only one document (if present) and returns True or False.
---
 mongoengine/queryset/base.py |  9 +++++++++
 tests/queryset/queryset.py   | 23 +++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index c2ad027e..823bc164 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -154,6 +154,15 @@ class BaseQuerySet(object):
     def __iter__(self):
         raise NotImplementedError
 
+    def __nonzero__(self):
+        """ Avoid to open all records in an if stmt """
+
+        for value in self:
+            self.rewind()
+            return True
+
+        return False        
+
     # Core functions
 
     def all(self):
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 7ff2965d..f274e0ee 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -3814,6 +3814,29 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(Example.objects(size=instance_size).count(), 1)
         self.assertEqual(Example.objects(size__in=[instance_size]).count(), 1)
 
+    def test_cursor_in_an_if_stmt(self):
+
+        class Test(Document):
+            test_field = StringField()
+
+        Test.drop_collection()
+        queryset = Test.objects
+
+        if queryset:
+            raise AssertionError('Empty cursor returns True')
+
+        test = Test()
+        test.test_field = 'test'
+        test.save()
+
+        queryset = Test.objects
+        if not test:
+            raise AssertionError('There is data, but cursor returned False')
+
+        queryset.next()
+        if queryset:
+            raise AssertionError('There is no data left in cursor')
+
 
 if __name__ == '__main__':
     unittest.main()

From 9ba657797ed64d50a876c95bc2a243bed3037e19 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Wed, 28 May 2014 08:33:22 -0300
Subject: [PATCH 23/73] Authors updated according guideline

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d6994d50..5d6a45fa 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -189,3 +189,4 @@ that much better:
  * Tom (https://github.com/tomprimozic)
  * j0hnsmith (https://github.com/j0hnsmith)
  * Damien Churchill (https://github.com/damoxc)
+ * Jonathan Simon Prates (https://github.com/jonathansp)
\ No newline at end of file

From 47f0de9836a7e340047c6639dee9ade29b9755cb Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Wed, 28 May 2014 08:36:57 -0300
Subject: [PATCH 24/73] Py3 fix

---
 mongoengine/queryset/base.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 823bc164..022b7c1f 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -161,7 +161,12 @@ class BaseQuerySet(object):
             self.rewind()
             return True
 
-        return False        
+        return False
+
+    def __bool__(self):
+        """ Same behaviour in Py3 """
+
+        return self.__nonzero__():
 
     # Core functions
 

From edcdfeb0573f253227a196b3080344e849f48109 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Wed, 28 May 2014 09:03:12 -0300
Subject: [PATCH 25/73] Fix syntax error

---
 mongoengine/queryset/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 022b7c1f..0dfff7cc 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -166,7 +166,7 @@ class BaseQuerySet(object):
     def __bool__(self):
         """ Same behaviour in Py3 """
 
-        return self.__nonzero__():
+        return self.__nonzero__()
 
     # Core functions
 

From dfdecef8e72fdd0379cdec4452f3175f0826b892 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Wed, 28 May 2014 09:40:22 -0300
Subject: [PATCH 26/73] Fix py2 and py3

---
 mongoengine/queryset/base.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 0dfff7cc..09fb5bf6 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -154,7 +154,7 @@ class BaseQuerySet(object):
     def __iter__(self):
         raise NotImplementedError
 
-    def __nonzero__(self):
+    def _bool(self):
         """ Avoid to open all records in an if stmt """
 
         for value in self:
@@ -163,10 +163,15 @@ class BaseQuerySet(object):
 
         return False
 
+    def __nonzero__(self):
+        """ Same behaviour in Py2 """
+
+        return self._bool()
+
     def __bool__(self):
         """ Same behaviour in Py3 """
 
-        return self.__nonzero__()
+        return self._bool()
 
     # Core functions
 

From ee0c7fd8bfa08ec885f379b2aa6dd72f48d8d7c5 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Wed, 28 May 2014 13:21:00 -0300
Subject: [PATCH 27/73] Change for loop to self.first()

---
 mongoengine/queryset/base.py | 6 +-----
 tests/queryset/queryset.py   | 4 ++--
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 09fb5bf6..099831fe 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -157,11 +157,7 @@ class BaseQuerySet(object):
     def _bool(self):
         """ Avoid to open all records in an if stmt """
 
-        for value in self:
-            self.rewind()
-            return True
-
-        return False
+        return False if self.first() is None else True
 
     def __nonzero__(self):
         """ Same behaviour in Py2 """
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index f274e0ee..f6adaf39 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -3834,8 +3834,8 @@ class QuerySetTest(unittest.TestCase):
             raise AssertionError('There is data, but cursor returned False')
 
         queryset.next()
-        if queryset:
-            raise AssertionError('There is no data left in cursor')
+        if not queryset:
+            raise AssertionError('There is data, cursor must return True')
 
 
 if __name__ == '__main__':

From 30964f65e465f7e29960cd49caf29d5c9d4ac756 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Wed, 28 May 2014 17:06:15 -0300
Subject: [PATCH 28/73] Remove orderby in if stmt

---
 mongoengine/queryset/base.py |  8 +++++-
 tests/queryset/queryset.py   | 53 ++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 099831fe..fe1285c6 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -157,7 +157,13 @@ class BaseQuerySet(object):
     def _bool(self):
         """ Avoid to open all records in an if stmt """
 
-        return False if self.first() is None else True
+        queryset = self.clone()
+        queryset._ordering = []
+        try:
+            queryset[0]
+            return True
+        except IndexError:
+            return False
 
     def __nonzero__(self):
         """ Same behaviour in Py2 """
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index f6adaf39..65f7255b 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -3837,6 +3837,59 @@ class QuerySetTest(unittest.TestCase):
         if not queryset:
             raise AssertionError('There is data, cursor must return True')
 
+    def test_bool_performance(self):
+
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        for i in xrange(100):
+            Person(name="No: %s" % i).save()
+
+        with query_counter() as q:
+            if Person.objects:
+                pass
+
+            self.assertEqual(q, 1)
+            op = q.db.system.profile.find({"ns":
+                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+
+            self.assertEqual(op['nreturned'], 1)
+
+
+    def test_bool_with_ordering(self):
+
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        Person(name="Test").save()
+
+        qs = Person.objects.order_by('name')
+
+        with query_counter() as q:
+
+            if qs:
+                pass
+
+            op = q.db.system.profile.find({"ns": 
+                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+
+            self.assertFalse('$orderby' in op['query'],
+                'BaseQuerySet cannot use orderby in if stmt')
+
+        with query_counter() as p:
+
+            for x in qs:
+                pass
+
+            op = p.db.system.profile.find({"ns":
+                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+
+
+            self.assertTrue('$orderby' in op['query'],
+                'BaseQuerySet cannot remove orderby in for loop')
+
 
 if __name__ == '__main__':
     unittest.main()

From 39735594bd935f3003d85009c3dc14a8d3d71f23 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Wed, 28 May 2014 17:15:48 -0300
Subject: [PATCH 29/73] Removed blank line

---
 tests/queryset/queryset.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 65f7255b..c5fea003 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -3886,7 +3886,6 @@ class QuerySetTest(unittest.TestCase):
             op = p.db.system.profile.find({"ns":
                 {"$ne": "%s.system.indexes" % q.db.name}})[0]
 
-
             self.assertTrue('$orderby' in op['query'],
                 'BaseQuerySet cannot remove orderby in for loop')
 

From c87801f0a911bd02f4b4ae910751c8b04717fd02 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Wed, 28 May 2014 17:26:28 -0300
Subject: [PATCH 30/73] Using first() from cloned queryset

---
 mongoengine/queryset/base.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index fe1285c6..85d2dc3f 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -159,11 +159,7 @@ class BaseQuerySet(object):
 
         queryset = self.clone()
         queryset._ordering = []
-        try:
-            queryset[0]
-            return True
-        except IndexError:
-            return False
+        return False if queryset.first() is None else True
 
     def __nonzero__(self):
         """ Same behaviour in Py2 """

From c744104a18829f7f34d491a580914b89ecf3c620 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Thu, 29 May 2014 10:53:20 -0300
Subject: [PATCH 31/73] Added test with meta

---
 tests/queryset/queryset.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index c5fea003..bad0d1e5 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -3889,6 +3889,33 @@ class QuerySetTest(unittest.TestCase):
             self.assertTrue('$orderby' in op['query'],
                 'BaseQuerySet cannot remove orderby in for loop')
 
+    def test_bool_with_ordering_from_meta_dict(self):
+
+        class Person(Document):
+            name = StringField()
+            meta = {
+                'ordering': ['name']
+            }
+
+        Person.drop_collection()
+        
+        Person(name="B").save()
+        Person(name="C").save()
+        Person(name="A").save()
+
+        with query_counter() as q:
+
+            if Person.objects:
+                pass
+
+            op = q.db.system.profile.find({"ns": 
+                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+
+            self.assertTrue('$orderby' in op['query'],
+                'BaseQuerySet cannot remove orderby from meta in boolen test')
+
+            self.assertEqual(Person.objects.first().name, 'A') 
+
 
 if __name__ == '__main__':
     unittest.main()

From 819ff2a90286030740e2d6d886c2f14e29d0ccc2 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Thu, 29 May 2014 14:36:30 -0300
Subject: [PATCH 32/73] Renamed to has_data()

---
 AUTHORS                      |  3 ++-
 mongoengine/queryset/base.py | 12 ++++++------
 tests/queryset/queryset.py   |  8 ++++++--
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 5d6a45fa..7b466d6b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -189,4 +189,5 @@ that much better:
  * Tom (https://github.com/tomprimozic)
  * j0hnsmith (https://github.com/j0hnsmith)
  * Damien Churchill (https://github.com/damoxc)
- * Jonathan Simon Prates (https://github.com/jonathansp)
\ No newline at end of file
+ * Jonathan Simon Prates (https://github.com/jonathansp)
+ * Thiago Papageorgiou (https://github.com/tmpapageorgiou)
\ No newline at end of file
diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 85d2dc3f..ef8cd2a7 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -154,22 +154,22 @@ class BaseQuerySet(object):
     def __iter__(self):
         raise NotImplementedError
 
-    def _bool(self):
-        """ Avoid to open all records in an if stmt """
+    def has_data(self):
+        """ Retrieves whether cursor has any data. """
 
         queryset = self.clone()
         queryset._ordering = []
         return False if queryset.first() is None else True
 
     def __nonzero__(self):
-        """ Same behaviour in Py2 """
+        """ Avoid to open all records in an if stmt in Py2. """
 
-        return self._bool()
+        return self.has_data()
 
     def __bool__(self):
-        """ Same behaviour in Py3 """
+        """ Avoid to open all records in an if stmt in Py3. """
 
-        return self._bool()
+        return self.has_data()
 
     # Core functions
 
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index bad0d1e5..fe932367 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -3831,11 +3831,15 @@ class QuerySetTest(unittest.TestCase):
 
         queryset = Test.objects
         if not test:
-            raise AssertionError('There is data, but cursor returned False')
+            raise AssertionError('Cursor has data and returned False')
 
         queryset.next()
         if not queryset:
-            raise AssertionError('There is data, cursor must return True')
+            raise AssertionError('Cursor has data and it must returns False,'
+                ' even in the last item.')
+
+        self.assertTrue(queryset.has_data(), 'Cursor has data and '
+            'returned False')
 
     def test_bool_performance(self):
 

From 85187239b629772ba276aaee54eb678c64ad8207 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Thu, 29 May 2014 15:21:24 -0300
Subject: [PATCH 33/73] Fix tests msg

---
 tests/queryset/queryset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index fe932367..f68468ff 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -3835,7 +3835,7 @@ class QuerySetTest(unittest.TestCase):
 
         queryset.next()
         if not queryset:
-            raise AssertionError('Cursor has data and it must returns False,'
+            raise AssertionError('Cursor has data and it must returns True,'
                 ' even in the last item.')
 
         self.assertTrue(queryset.has_data(), 'Cursor has data and '

From 1eacc6fbff0bbe14a18d62032fe0b616194c8d26 Mon Sep 17 00:00:00 2001
From: Stefan Wojcik <wojcikstefan@gmail.com>
Date: Fri, 30 May 2014 15:08:03 -0700
Subject: [PATCH 34/73] clear ordering via empty order_by

---
 mongoengine/queryset/base.py |  7 ++--
 tests/queryset/queryset.py   | 70 ++++++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 3 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index c2ad027e..4f451667 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -50,7 +50,7 @@ class BaseQuerySet(object):
         self._initial_query = {}
         self._where_clause = None
         self._loaded_fields = QueryFieldList()
-        self._ordering = []
+        self._ordering = None
         self._snapshot = False
         self._timeout = True
         self._class_check = True
@@ -1189,8 +1189,9 @@ class BaseQuerySet(object):
             if self._ordering:
                 # Apply query ordering
                 self._cursor_obj.sort(self._ordering)
-            elif self._document._meta['ordering']:
-                # Otherwise, apply the ordering from the document model
+            elif self._ordering is None and self._document._meta['ordering']:
+                # Otherwise, apply the ordering from the document model, unless
+                # it's been explicitly cleared via order_by with no arguments
                 order = self._get_order_by(self._document._meta['ordering'])
                 self._cursor_obj.sort(order)
 
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 7ff2965d..d706eda8 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1040,6 +1040,76 @@ class QuerySetTest(unittest.TestCase):
         expected = [blog_post_1, blog_post_2, blog_post_3]
         self.assertSequence(qs, expected)
 
+    def test_clear_ordering(self):
+        """ Make sure one can clear the query set ordering by applying a
+        consecutive order_by()
+        """
+
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        Person(name="A").save()
+        Person(name="B").save()
+
+        qs = Person.objects.order_by('-name')
+
+        # Make sure we can clear a previously specified ordering
+        with query_counter() as q:
+            lst = list(qs.order_by())
+
+            op = q.db.system.profile.find({"ns":
+                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+
+            self.assertTrue('$orderby' not in op['query'])
+            self.assertEqual(lst[0].name, 'A')
+
+        # Make sure previously specified ordering is preserved during
+        # consecutive calls to the same query set
+        with query_counter() as q:
+            lst = list(qs)
+
+            op = q.db.system.profile.find({"ns":
+                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+
+            self.assertTrue('$orderby' in op['query'])
+            self.assertEqual(lst[0].name, 'B')
+
+    def test_clear_default_ordering(self):
+
+        class Person(Document):
+            name = StringField()
+            meta = {
+                'ordering': ['-name']
+            }
+
+        Person.drop_collection()
+        Person(name="A").save()
+        Person(name="B").save()
+
+        qs = Person.objects
+
+        # Make sure clearing default ordering works
+        with query_counter() as q:
+            lst = list(qs.order_by())
+
+            op = q.db.system.profile.find({"ns":
+                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+
+            self.assertTrue('$orderby' not in op['query'])
+            self.assertEqual(lst[0].name, 'A')
+
+        # Make sure default ordering is preserved during consecutive calls
+        # to the same query set
+        with query_counter() as q:
+            lst = list(qs)
+
+            op = q.db.system.profile.find({"ns":
+                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+
+            self.assertTrue('$orderby' in op['query'])
+            self.assertEqual(lst[0].name, 'B')
+
     def test_find_embedded(self):
         """Ensure that an embedded document is properly returned from a query.
         """

From 9835b382dab4ed3ceef277a2948dde103542303c Mon Sep 17 00:00:00 2001
From: Sagiv Malihi <sagivmalihi@gmail.com>
Date: Thu, 3 Apr 2014 12:38:33 +0300
Subject: [PATCH 35/73] added __slots__ to BaseDocument and Document changed
 the _data field to static key-value mapping instead of hash table This
 implements #624

---
 mongoengine/base/datastructures.py |  97 ++++++++++++++++++++++++++
 mongoengine/base/document.py       |  46 +++++++++----
 mongoengine/document.py            |   7 +-
 tests/test_datastructures.py       | 107 +++++++++++++++++++++++++++++
 4 files changed, 243 insertions(+), 14 deletions(-)
 create mode 100644 tests/test_datastructures.py

diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py
index 4652fb56..32a66018 100644
--- a/mongoengine/base/datastructures.py
+++ b/mongoengine/base/datastructures.py
@@ -1,4 +1,6 @@
 import weakref
+import functools
+import itertools
 from mongoengine.common import _import_class
 
 __all__ = ("BaseDict", "BaseList")
@@ -156,3 +158,98 @@ class BaseList(list):
     def _mark_as_changed(self):
         if hasattr(self._instance, '_mark_as_changed'):
             self._instance._mark_as_changed(self._name)
+
+
+class StrictDict(object):
+    __slots__ = () 
+    _special_fields = set(['get', 'pop', 'iteritems', 'items', 'keys', 'create'])
+    _classes = {}
+    def __init__(self, **kwargs):
+        for k,v in kwargs.iteritems():
+            setattr(self, k, v)
+    def __getitem__(self, key):
+        key = '_reserved_' + key if key in self._special_fields else key
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            raise KeyError(key)
+    def __setitem__(self, key, value):
+        key = '_reserved_' + key if key in self._special_fields else key
+        return setattr(self, key, value)
+    def __contains__(self, key):
+        return hasattr(self, key)
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+    def pop(self, key, default=None):
+        v = self.get(key, default)
+        try:
+            delattr(self, key)
+        except AttributeError:
+            pass
+        return v
+    def iteritems(self):
+        for key in self:
+            yield key, self[key]
+    def items(self):
+        return [(k, self[k]) for k in iter(self)]
+    def keys(self):
+        return list(iter(self))
+    def __iter__(self):
+        return (key for key in self.__slots__ if hasattr(self, key))
+    def __len__(self):
+        return len(list(self.iteritems()))
+    def __eq__(self, other):
+        return self.items() == other.items()
+    def __neq__(self, other):
+        return self.items() != other.items()
+    
+    @classmethod
+    def create(cls, allowed_keys):
+        allowed_keys_tuple = tuple(('_reserved_' + k if k in cls._special_fields else k) for k in allowed_keys)
+        allowed_keys = frozenset(allowed_keys_tuple)
+        if allowed_keys not in cls._classes:
+            class SpecificStrictDict(cls):
+                __slots__ = allowed_keys_tuple
+            cls._classes[allowed_keys] = SpecificStrictDict 
+        return cls._classes[allowed_keys]
+
+
+class SemiStrictDict(StrictDict):
+    __slots__ = ('_extras')
+    _classes = {}
+    def __getattr__(self, attr):
+        try:
+            super(SemiStrictDict, self).__getattr__(attr)
+        except AttributeError:
+            try:
+                return self.__getattribute__('_extras')[attr]
+            except KeyError as e:
+                raise AttributeError(e)
+    def __setattr__(self, attr, value):
+        try:
+            super(SemiStrictDict, self).__setattr__(attr, value)
+        except AttributeError:
+            try:
+                self._extras[attr] = value
+            except AttributeError:
+                self._extras = {attr: value}
+
+    def __delattr__(self, attr):
+        try:
+            super(SemiStrictDict, self).__delattr__(attr)
+        except AttributeError:
+            try:
+                del self._extras[attr]
+            except KeyError as e:
+                raise AttributeError(e)
+
+    def __iter__(self):
+        try:
+            extras_iter = iter(self.__getattribute__('_extras'))
+        except AttributeError:
+            extras_iter = ()
+        return itertools.chain(super(SemiStrictDict, self).__iter__(), extras_iter)
+
diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index cea2f09b..01809aa9 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -16,20 +16,20 @@ from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type,
                                         to_str_keys_recursive)
 
 from mongoengine.base.common import get_document, ALLOW_INHERITANCE
-from mongoengine.base.datastructures import BaseDict, BaseList
+from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict, SemiStrictDict 
 from mongoengine.base.fields import ComplexBaseField
 
 __all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
 
 NON_FIELD_ERRORS = '__all__'
 
-
 class BaseDocument(object):
+    __slots__ = ('_changed_fields', '_initialised', '_created', '_data',
+                  '_dynamic_fields', '_auto_id_field', '_db_field_map', '_cls', '__weakref__')
 
     _dynamic = False
-    _created = True
     _dynamic_lock = True
-    _initialised = False
+    STRICT = False
 
     def __init__(self, *args, **values):
         """
@@ -38,6 +38,8 @@ class BaseDocument(object):
         :param __auto_convert: Try and will cast python objects to Object types
         :param values: A dictionary of values for the document
         """
+        self._initialised = False
+        self._created = True
         if args:
             # Combine positional arguments with named arguments.
             # We only want named arguments.
@@ -52,8 +54,12 @@ class BaseDocument(object):
                 values[name] = value
         __auto_convert = values.pop("__auto_convert", True)
         signals.pre_init.send(self.__class__, document=self, values=values)
-
-        self._data = {}
+        
+        if self.STRICT and not self._dynamic:
+            self._data = StrictDict.create(allowed_keys=self._fields.keys())()
+        else:
+            self._data = SemiStrictDict.create(allowed_keys=self._fields.keys())()
+            
         self._dynamic_fields = SON()
 
         # Assign default values to instance
@@ -129,17 +135,25 @@ class BaseDocument(object):
                 self._data[name] = value
                 if hasattr(self, '_changed_fields'):
                     self._mark_as_changed(name)
+        try:
+            self__created = self._created
+        except AttributeError:
+            self__created = True
 
-        if (self._is_document and not self._created and
+        if (self._is_document and not self__created and
            name in self._meta.get('shard_key', tuple()) and
            self._data.get(name) != value):
             OperationError = _import_class('OperationError')
             msg = "Shard Keys are immutable. Tried to update %s" % name
             raise OperationError(msg)
 
+        try:
+            self__initialised = self._initialised
+        except AttributeError:
+            self__initialised = False 
         # Check if the user has created a new instance of a class
-        if (self._is_document and self._initialised
-           and self._created and name == self._meta['id_field']):
+        if (self._is_document and self__initialised
+           and self__created and name == self._meta['id_field']):
                 super(BaseDocument, self).__setattr__('_created', False)
 
         super(BaseDocument, self).__setattr__(name, value)
@@ -157,9 +171,11 @@ class BaseDocument(object):
         if isinstance(data["_data"], SON):
             data["_data"] = self.__class__._from_son(data["_data"])._data
         for k in ('_changed_fields', '_initialised', '_created', '_data',
-                  '_fields_ordered', '_dynamic_fields'):
+                   '_dynamic_fields'):
             if k in data:
                 setattr(self, k, data[k])
+        if '_fields_ordered' in data:
+            setattr(type(self), '_fields_ordered', data['_fields_ordered'])
         dynamic_fields = data.get('_dynamic_fields') or SON()
         for k in dynamic_fields.keys():
             setattr(self, k, data["_data"].get(k))
@@ -576,7 +592,9 @@ class BaseDocument(object):
             msg = ("Invalid data to create a `%s` instance.\n%s"
                    % (cls._class_name, errors))
             raise InvalidDocumentError(msg)
-
+        
+        if cls.STRICT:
+            data = dict((k, v) for k,v in data.iteritems() if k in cls._fields)
         obj = cls(__auto_convert=False, **data)
         obj._changed_fields = changed_fields
         obj._created = False
@@ -813,7 +831,11 @@ class BaseDocument(object):
         """Dynamically set the display value for a field with choices"""
         for attr_name, field in self._fields.items():
             if field.choices:
-                setattr(self,
+                if self._dynamic:
+                    obj = self
+                else:
+                    obj = type(self)
+                setattr(obj,
                         'get_%s_display' % attr_name,
                         partial(self.__get_field_display, field=field))
 
diff --git a/mongoengine/document.py b/mongoengine/document.py
index 1bbd7b73..98e1d2a3 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -52,16 +52,17 @@ class EmbeddedDocument(BaseDocument):
     `_cls` set :attr:`allow_inheritance` to ``False`` in the :attr:`meta`
     dictionary.
     """
+    
+    __slots__ = ('_instance')
 
     # The __metaclass__ attribute is removed by 2to3 when running with Python3
     # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
     my_metaclass  = DocumentMetaclass
     __metaclass__ = DocumentMetaclass
 
-    _instance = None
-
     def __init__(self, *args, **kwargs):
         super(EmbeddedDocument, self).__init__(*args, **kwargs)
+        self._instance = None
         self._changed_fields = []
 
     def __eq__(self, other):
@@ -124,6 +125,8 @@ class Document(BaseDocument):
     my_metaclass  = TopLevelDocumentMetaclass
     __metaclass__ = TopLevelDocumentMetaclass
 
+    __slots__ = ('__objects' )
+
     def pk():
         """Primary key alias
         """
diff --git a/tests/test_datastructures.py b/tests/test_datastructures.py
new file mode 100644
index 00000000..c761a41e
--- /dev/null
+++ b/tests/test_datastructures.py
@@ -0,0 +1,107 @@
+import unittest
+from mongoengine.base.datastructures import StrictDict, SemiStrictDict 
+
+class TestStrictDict(unittest.TestCase):
+    def strict_dict_class(self, *args, **kwargs):
+        return StrictDict.create(*args, **kwargs)
+    def setUp(self):
+        self.dtype = self.strict_dict_class(("a", "b", "c"))
+    def test_init(self):
+        d = self.dtype(a=1, b=1, c=1)
+        self.assertEqual((d.a, d.b, d.c), (1, 1, 1))
+
+    def test_init_fails_on_nonexisting_attrs(self):
+        self.assertRaises(AttributeError, lambda: self.dtype(a=1, b=2, d=3))
+        
+    def test_eq(self):
+        d = self.dtype(a=1, b=1, c=1)
+        dd = self.dtype(a=1, b=1, c=1)
+        e = self.dtype(a=1, b=1, c=3)
+        f = self.dtype(a=1, b=1)
+        g = self.strict_dict_class(("a", "b", "c", "d"))(a=1, b=1, c=1, d=1)
+        h = self.strict_dict_class(("a", "c", "b"))(a=1, b=1, c=1)
+        i = self.strict_dict_class(("a", "c", "b"))(a=1, b=1, c=2)
+        
+        self.assertEqual(d, dd)
+        self.assertNotEqual(d, e)
+        self.assertNotEqual(d, f)
+        self.assertNotEqual(d, g)
+        self.assertNotEqual(f, d)
+        self.assertEqual(d, h)
+        self.assertNotEqual(d, i)
+
+    def test_setattr_getattr(self):
+        d = self.dtype()
+        d.a = 1
+        self.assertEqual(d.a, 1)
+        self.assertRaises(AttributeError, lambda: d.b)
+    
+    def test_setattr_raises_on_nonexisting_attr(self):
+        d = self.dtype()
+        def _f():
+            d.x=1
+        self.assertRaises(AttributeError, _f)
+    
+    def test_setattr_getattr_special(self):
+        d = self.strict_dict_class(["items"])
+        d.items = 1
+        self.assertEqual(d.items, 1)
+    
+    def test_get(self):
+        d = self.dtype(a=1)
+        self.assertEqual(d.get('a'), 1)
+        self.assertEqual(d.get('b', 'bla'), 'bla')
+
+    def test_items(self):
+        d = self.dtype(a=1)
+        self.assertEqual(d.items(), [('a', 1)])
+        d = self.dtype(a=1, b=2)
+        self.assertEqual(d.items(), [('a', 1), ('b', 2)])
+
+    def test_mappings_protocol(self):
+        d = self.dtype(a=1, b=2)
+        assert dict(d) == {'a': 1, 'b': 2}
+        assert dict(**d) == {'a': 1, 'b': 2}
+
+
+class TestSemiSrictDict(TestStrictDict):
+    def strict_dict_class(self, *args, **kwargs):
+        return SemiStrictDict.create(*args, **kwargs)
+
+    def test_init_fails_on_nonexisting_attrs(self):
+        # disable irrelevant test
+        pass
+
+    def test_setattr_raises_on_nonexisting_attr(self):
+        # disable irrelevant test
+        pass
+
+    def test_setattr_getattr_nonexisting_attr_succeeds(self):
+        d = self.dtype()
+        d.x = 1
+        self.assertEqual(d.x, 1)
+
+    def test_init_succeeds_with_nonexisting_attrs(self):
+        d = self.dtype(a=1, b=1, c=1, x=2)
+        self.assertEqual((d.a, d.b, d.c, d.x), (1, 1, 1, 2))
+   
+    def test_iter_with_nonexisting_attrs(self):
+        d = self.dtype(a=1, b=1, c=1, x=2)
+        self.assertEqual(list(d), ['a', 'b', 'c', 'x'])
+
+    def test_iteritems_with_nonexisting_attrs(self):
+        d = self.dtype(a=1, b=1, c=1, x=2)
+        self.assertEqual(list(d.iteritems()), [('a', 1), ('b', 1), ('c', 1), ('x', 2)])
+
+    def tets_cmp_with_strict_dicts(self):
+        d = self.dtype(a=1, b=1, c=1)
+        dd = StrictDict.create(("a", "b", "c"))(a=1, b=1, c=1)
+        self.assertEqual(d, dd)
+
+    def test_cmp_with_strict_dict_with_nonexisting_attrs(self):
+        d = self.dtype(a=1, b=1, c=1, x=2)
+        dd = StrictDict.create(("a", "b", "c", "x"))(a=1, b=1, c=1, x=2)
+        self.assertEqual(d, dd)
+
+if __name__ == '__main__':
+    unittest.main()

From 7bb2fe128a4b8c8b08f4d800f15229e2814bfac8 Mon Sep 17 00:00:00 2001
From: Jonathan Prates <jonathan.prates@corp.terra.com.br>
Date: Thu, 12 Jun 2014 11:08:41 -0300
Subject: [PATCH 36/73] Added PR #657

---
 mongoengine/queryset/base.py | 11 +++++------
 tests/queryset/queryset.py   |  9 ++++-----
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 94a6e4b5..cb48f6ca 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -154,22 +154,21 @@ class BaseQuerySet(object):
     def __iter__(self):
         raise NotImplementedError
 
-    def has_data(self):
+    def _has_data(self):
         """ Retrieves whether cursor has any data. """
 
-        queryset = self.clone()
-        queryset._ordering = []
+        queryset = self.order_by()
         return False if queryset.first() is None else True
 
     def __nonzero__(self):
         """ Avoid to open all records in an if stmt in Py2. """
 
-        return self.has_data()
+        return self._has_data()
 
     def __bool__(self):
         """ Avoid to open all records in an if stmt in Py3. """
 
-        return self.has_data()
+        return self._has_data()
 
     # Core functions
 
@@ -1410,7 +1409,7 @@ class BaseQuerySet(object):
                 pass
             key_list.append((key, direction))
 
-        if self._cursor_obj:
+        if self._cursor_obj and key_list:
             self._cursor_obj.sort(key_list)
         return key_list
 
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 38499df4..a2438e21 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -3908,9 +3908,6 @@ class QuerySetTest(unittest.TestCase):
             raise AssertionError('Cursor has data and it must returns True,'
                 ' even in the last item.')
 
-        self.assertTrue(queryset.has_data(), 'Cursor has data and '
-            'returned False')
-
     def test_bool_performance(self):
 
         class Person(Document):
@@ -3985,10 +3982,12 @@ class QuerySetTest(unittest.TestCase):
             op = q.db.system.profile.find({"ns": 
                 {"$ne": "%s.system.indexes" % q.db.name}})[0]
 
-            self.assertTrue('$orderby' in op['query'],
-                'BaseQuerySet cannot remove orderby from meta in boolen test')
+            self.assertFalse('$orderby' in op['query'],
+                'BaseQuerySet must remove orderby from meta in boolen test')
 
             self.assertEqual(Person.objects.first().name, 'A') 
+            self.assertTrue(Person.objects._has_data(),
+                            'Cursor has data and returned False')
 
 
 if __name__ == '__main__':

From 03559a3cc4b0ee3a5509c8eb2f1f54bf342284b4 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gmail.com>
Date: Tue, 24 Jun 2014 19:20:15 +0300
Subject: [PATCH 37/73] Added Python 3.4 to the build process.

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 5739909b..7ba22694 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,7 @@ python:
     - "2.7"
     - "3.2"
     - "3.3"
+    - "3.4"
 env:
   - PYMONGO=dev DJANGO=1.6
   - PYMONGO=dev DJANGO=1.5.5

From bb461b009f6aa0c64c6fb2cf845a4714ee324908 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gmail.com>
Date: Tue, 24 Jun 2014 19:39:27 +0300
Subject: [PATCH 38/73] Travis build improvements.

The latest patch version of each Django minor version is used.
The build now installs existing pymongo versions.
The build now actually tests against the specified Django version.
Replaced PIL with Pillow.
Added PyPy and Python 3.4 to the build.

Rebase Log:

Installing Pillow instead of PIL for testing since it's recommended and it supports PyPy.

Excluding Django versions that do not work with Python 3.

Improved formatting of .travis.yml.

Specifying Pillow 2.0.0 and above since it's the first version that is supported in Python 3.

PIL should not be installed alongside Pillow.

Also, I installed some libraries that both PIL and Pillow depend on.

It seems I have to be explicit on all envvars in order to exclude Django 1.4 from the build matrix.

The build is now installing pymongo versions that actually exist.

openjpeg has a different name on Ubuntu 12.04.

Restoring libz hack.

Also installing all Pillow requirements just in case.

Fixed the build matrix.

Acting according to @BanzaiMan's advice in travis-ci/travis-ci/#1492.
---
 .travis.yml | 58 +++++++++++++++++++++++++++++++++++++++++------------
 setup.py    |  4 ++--
 2 files changed, 47 insertions(+), 15 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5739909b..d00dde3a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,22 +6,54 @@ python:
     - "2.7"
     - "3.2"
     - "3.3"
+    - "3.4"
+    - "pypy"
 env:
-  - PYMONGO=dev DJANGO=1.6
-  - PYMONGO=dev DJANGO=1.5.5
-  - PYMONGO=dev DJANGO=1.4.10
-  - PYMONGO=2.5 DJANGO=1.6
-  - PYMONGO=2.5 DJANGO=1.5.5
-  - PYMONGO=2.5 DJANGO=1.4.10
-  - PYMONGO=3.2 DJANGO=1.6
-  - PYMONGO=3.2 DJANGO=1.5.5
-  - PYMONGO=3.3 DJANGO=1.6
-  - PYMONGO=3.3 DJANGO=1.5.5
+  - PYMONGO=dev DJANGO=1.6.5
+  - PYMONGO=dev DJANGO=1.5.8
+  - PYMONGO=dev DJANGO=1.4.13
+  - PYMONGO=2.5.2 DJANGO=1.6.5
+  - PYMONGO=2.5.2 DJANGO=1.5.8
+  - PYMONGO=2.5.2 DJANGO=1.4.13
+  - PYMONGO=2.6.3 DJANGO=1.6.5
+  - PYMONGO=2.6.3 DJANGO=1.5.8
+  - PYMONGO=2.6.3 DJANGO=1.4.13
+  - PYMONGO=2.7.1 DJANGO=1.6.5
+  - PYMONGO=2.7.1 DJANGO=1.5.8
+  - PYMONGO=2.7.1 DJANGO=1.4.13
+  
+matrix:
+    exclude:
+        - python: "3.2"
+          env: PYMONGO=dev DJANGO=1.4.13
+        - python: "3.2"
+          env: PYMONGO=2.5.2 DJANGO=1.4.13
+        - python: "3.2"
+          env: PYMONGO=2.6.3 DJANGO=1.4.13
+        - python: "3.2"
+          env: PYMONGO=2.7.1 DJANGO=1.4.13
+        - python: "3.3"
+          env: PYMONGO=dev DJANGO=1.4.13
+        - python: "3.3"
+          env: PYMONGO=2.5.2 DJANGO=1.4.13
+        - python: "3.3"
+          env: PYMONGO=2.6.3 DJANGO=1.4.13
+        - python: "3.3"
+          env: PYMONGO=2.7.1 DJANGO=1.4.13
+        - python: "3.4"
+          env: PYMONGO=dev DJANGO=1.4.13
+        - python: "3.4"
+          env: PYMONGO=2.5.2 DJANGO=1.4.13
+        - python: "3.4"
+          env: PYMONGO=2.6.3 DJANGO=1.4.13
+        - python: "3.4"
+          env: PYMONGO=2.7.1 DJANGO=1.4.13
 install:
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi
+    - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk
+    - cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/
     - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi
-    - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi
+    - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO; true; fi
+    - pip install Django==$DJANGO
     - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b
     - python setup.py install
 script:
diff --git a/setup.py b/setup.py
index 85707d00..40c27ebe 100644
--- a/setup.py
+++ b/setup.py
@@ -51,12 +51,12 @@ CLASSIFIERS = [
 extra_opts = {"packages": find_packages(exclude=["tests", "tests.*"])}
 if sys.version_info[0] == 3:
     extra_opts['use_2to3'] = True
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1']
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'Pillow>=2.0.0', 'django>=1.5.1']
     if "test" in sys.argv or "nosetests" in sys.argv:
         extra_opts['packages'] = find_packages()
         extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]}
 else:
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil']
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'Pillow>=2.0.0', 'jinja2>=2.6', 'python-dateutil']
 
 setup(name='mongoengine',
       version=VERSION,

From 8e852bce02b14a0149c0e189642631edc2de31fe Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 10:58:00 +0300
Subject: [PATCH 39/73] Pillow provides a more descriptive error message,
 therefor the build failure.

---
 tests/fields/file_tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py
index 902b1512..7ae53e8a 100644
--- a/tests/fields/file_tests.py
+++ b/tests/fields/file_tests.py
@@ -279,7 +279,7 @@ class FileTest(unittest.TestCase):
                 t.image.put(f)
                 self.fail("Should have raised an invalidation error")
             except ValidationError, e:
-                self.assertEqual("%s" % e, "Invalid image: cannot identify image file")
+                self.assertEqual("%s" % e, "Invalid image: cannot identify image file %s" % f)
 
         t = TestImage()
         t.image.put(open(TEST_IMAGE_PATH, 'rb'))

From adbbc656d41dbc7ecd33ae933c164efe3fd89316 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 11:12:40 +0300
Subject: [PATCH 40/73] Removing zlib hack since only PIL needs it. The build
 should pass without it.

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index d00dde3a..1265ba70 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -50,7 +50,6 @@ matrix:
           env: PYMONGO=2.7.1 DJANGO=1.4.13
 install:
     - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk
-    - cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/
     - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi
     - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO; true; fi
     - pip install Django==$DJANGO

From 8adf1cdd0212e18d26220d7e5997c3534c92a017 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 11:18:35 +0300
Subject: [PATCH 41/73] Fast finish the build if there are failures since we
 have a very large build matrix and each build takes a very long time.

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 1265ba70..4079855d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,6 +23,7 @@ env:
   - PYMONGO=2.7.1 DJANGO=1.4.13
   
 matrix:
+    fast_finish: true
     exclude:
         - python: "3.2"
           env: PYMONGO=dev DJANGO=1.4.13

From fc3eda55c72e99f0b80bb37bf917099780c2f79e Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 11:32:41 +0300
Subject: [PATCH 42/73] Added a note about optional dependencies to the README
 file.

---
 README.rst | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index cc4524ae..8c3ee26e 100644
--- a/README.rst
+++ b/README.rst
@@ -29,9 +29,18 @@ setup.py install``.
 
 Dependencies
 ============
-- pymongo 2.5+
+- pymongo>=2.5
 - sphinx (optional - for documentation generation)
 
+Optional Dependencies
+---------------------
+- **Django Integration:** Django>=1.4.0 for Python 2.x or PyPy and Django>=1.5.0 for Python 3.x
+- **Image Fields**: Pillow>=2.0.0 or PIL (not recommended since MongoEngine is tested with Pillow)
+- dateutil>=2.1.0
+
+.. note
+   MongoEngine always runs it's test suite against the latest patch version of each dependecy. e.g.: Django 1.6.5
+
 Examples
 ========
 Some simple examples of what MongoEngine code looks like::

From fe2ef4e61c4bd44e5156542728b7755f30ef8148 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 11:39:08 +0300
Subject: [PATCH 43/73] Made the benchmark script compatitable with Python 3
 and ensured it runs on every build.

---
 .travis.yml  |  1 +
 benchmark.py | 41 +++++++++++++++++++++--------------------
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 4079855d..fff35ca9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -58,6 +58,7 @@ install:
     - python setup.py install
 script:
     - python setup.py test
+    - python benchmark.py
 notifications:
   irc: "irc.freenode.org#mongoengine"
 branches:
diff --git a/benchmark.py b/benchmark.py
index 16b2fd47..cb5e19d8 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -113,6 +113,7 @@ def main():
     4.68946313858
     ----------------------------------------------------------------------------------------------------
     """
+    print("Benchmarking...")
 
     setup = """
 from pymongo import MongoClient
@@ -138,10 +139,10 @@ myNoddys = noddy.find()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
+    print("-" * 100)
     print """Creating 10000 dictionaries - Pymongo"""
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
 from pymongo import MongoClient
@@ -161,10 +162,10 @@ myNoddys = noddy.find()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
+    print("-" * 100)
     print """Creating 10000 dictionaries - Pymongo write_concern={"w": 0}"""
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     setup = """
 from pymongo import MongoClient
@@ -190,10 +191,10 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - MongoEngine"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - MongoEngine""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
 for i in xrange(10000):
@@ -208,10 +209,10 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
+    print("-" * 100)
     print """Creating 10000 dictionaries without continual assign - MongoEngine"""
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
 for i in xrange(10000):
@@ -224,10 +225,10 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
+    print("-" * 100)
     print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True"""
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
 for i in xrange(10000):
@@ -240,10 +241,10 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
+    print("-" * 100)
     print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True"""
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
 for i in xrange(10000):
@@ -256,10 +257,10 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
 for i in xrange(10000):
@@ -272,11 +273,11 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()

From f44c8f120532f407b5eea6fed480183c8aa51758 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 13:11:32 +0300
Subject: [PATCH 44/73] Skipping a test that does not work on PyPy due to a
 PyPy bug/feature.

---
 tests/queryset/queryset.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 7ff2965d..33f7dd91 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -3586,7 +3586,13 @@ class QuerySetTest(unittest.TestCase):
 
             [x for x in people]
             self.assertEqual(100, len(people._result_cache))
-            self.assertEqual(None, people._len)
+
+            import platform
+
+            if platform.python_implementation() != "PyPy":
+                # PyPy evaluates __len__ when iterating with list comprehensions while CPython does not.
+                # This may be a bug in PyPy (PyPy/#1802) but it does not affect the behavior of MongoEngine.
+                self.assertEqual(None, people._len)
             self.assertEqual(q, 1)
 
             list(people)

From 1914032e35ef917ea95efc192b8280858ff78c82 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 14:20:54 +0300
Subject: [PATCH 45/73] Missed some of the print statements in the benchmarks
 script.

---
 benchmark.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/benchmark.py b/benchmark.py
index cb5e19d8..5613eaf8 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -210,7 +210,7 @@ myNoddys = Noddy.objects()
 """
 
     print("-" * 100)
-    print """Creating 10000 dictionaries without continual assign - MongoEngine"""
+    print("""Creating 10000 dictionaries without continual assign - MongoEngine""")
     t = timeit.Timer(stmt=stmt, setup=setup)
     print(t.timeit(1))
 
@@ -242,7 +242,7 @@ myNoddys = Noddy.objects()
 """
 
     print("-" * 100)
-    print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True"""
+    print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""")
     t = timeit.Timer(stmt=stmt, setup=setup)
     print(t.timeit(1))
 

From 7f7745071af87957cdc5705b409b96ed49a1e481 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 15:47:54 +0300
Subject: [PATCH 46/73] Found more print statements that were not turned into
 function calls.

---
 benchmark.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmark.py b/benchmark.py
index 5613eaf8..d33e58db 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -140,7 +140,7 @@ myNoddys = noddy.find()
 """
 
     print("-" * 100)
-    print """Creating 10000 dictionaries - Pymongo"""
+    print("""Creating 10000 dictionaries - Pymongo""")
     t = timeit.Timer(stmt=stmt, setup=setup)
     print(t.timeit(1))
 
@@ -163,7 +163,7 @@ myNoddys = noddy.find()
 """
 
     print("-" * 100)
-    print """Creating 10000 dictionaries - Pymongo write_concern={"w": 0}"""
+    print("""Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""")
     t = timeit.Timer(stmt=stmt, setup=setup)
     print(t.timeit(1))
 
@@ -226,7 +226,7 @@ myNoddys = Noddy.objects()
 """
 
     print("-" * 100)
-    print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True"""
+    print("""Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""")
     t = timeit.Timer(stmt=stmt, setup=setup)
     print(t.timeit(1))
 

From 29309dac9a926077962fc0778e5b2fbaf1d29cc2 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 16:53:24 +0300
Subject: [PATCH 47/73] Mongo clients with the same settings should be shared
 since they manage a connection pool. Also, I removed old code that was
 supposed to support Pymongo<2.1 which we don't support anymore.

---
 mongoengine/connection.py | 33 ++++++++++++++++++---------------
 tests/test_connection.py  | 11 +++++++++++
 2 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/mongoengine/connection.py b/mongoengine/connection.py
index 7cc626f4..d3efac62 100644
--- a/mongoengine/connection.py
+++ b/mongoengine/connection.py
@@ -93,20 +93,11 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
             raise ConnectionError(msg)
         conn_settings = _connection_settings[alias].copy()
 
-        if hasattr(pymongo, 'version_tuple'):  # Support for 2.1+
-            conn_settings.pop('name', None)
-            conn_settings.pop('slaves', None)
-            conn_settings.pop('is_slave', None)
-            conn_settings.pop('username', None)
-            conn_settings.pop('password', None)
-        else:
-            # Get all the slave connections
-            if 'slaves' in conn_settings:
-                slaves = []
-                for slave_alias in conn_settings['slaves']:
-                    slaves.append(get_connection(slave_alias))
-                conn_settings['slaves'] = slaves
-                conn_settings.pop('read_preference', None)
+        conn_settings.pop('name', None)
+        conn_settings.pop('slaves', None)
+        conn_settings.pop('is_slave', None)
+        conn_settings.pop('username', None)
+        conn_settings.pop('password', None)
 
         connection_class = MongoClient
         if 'replicaSet' in conn_settings:
@@ -119,7 +110,19 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
             connection_class = MongoReplicaSetClient
 
         try:
-            _connections[alias] = connection_class(**conn_settings)
+            connection = None
+            connection_settings_iterator = ((alias, settings.copy()) for alias, settings in _connection_settings.iteritems())
+            for alias, connection_settings in connection_settings_iterator:
+                connection_settings.pop('name', None)
+                connection_settings.pop('slaves', None)
+                connection_settings.pop('is_slave', None)
+                connection_settings.pop('username', None)
+                connection_settings.pop('password', None)
+                if conn_settings == connection_settings and _connections.get(alias, None):
+                    connection = _connections[alias]
+                    break
+
+            _connections[alias] = connection if connection else connection_class(**conn_settings)
         except Exception, e:
             raise ConnectionError("Cannot connect to database %s :\n%s" % (alias, e))
     return _connections[alias]
diff --git a/tests/test_connection.py b/tests/test_connection.py
index 96135bc5..bf615ceb 100644
--- a/tests/test_connection.py
+++ b/tests/test_connection.py
@@ -34,6 +34,17 @@ class ConnectionTest(unittest.TestCase):
         conn = get_connection('testdb')
         self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
 
+    def test_sharing_connections(self):
+        """Ensure that connections are shared when the connection settings are exactly the same
+        """
+        connect('mongoenginetest', alias='testdb1')
+
+        expected_connection = get_connection('testdb1')
+
+        connect('mongoenginetest', alias='testdb2')
+        actual_connection = get_connection('testdb2')
+        self.assertIs(expected_connection, actual_connection)
+
     def test_connect_uri(self):
         """Ensure that the connect() method works properly with uri's
         """

From b8d568761e0e3130ce37d5675f8b218b11b88e29 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 17:24:52 +0300
Subject: [PATCH 48/73] Getting rid of xrange since it's not in Python 3 and
 does not affect the benchmark.

---
 benchmark.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/benchmark.py b/benchmark.py
index d33e58db..53ecf32c 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -15,7 +15,7 @@ def cprofile_main():
     class Noddy(Document):
         fields = DictField()
 
-    for i in xrange(1):
+    for i in range(1):
         noddy = Noddy()
         for j in range(20):
             noddy.fields["key" + str(j)] = "value " + str(j)
@@ -128,7 +128,7 @@ connection = MongoClient()
 db = connection.timeit_test
 noddy = db.noddy
 
-for i in xrange(10000):
+for i in range(10000):
     example = {'fields': {}}
     for j in range(20):
         example['fields']["key"+str(j)] = "value "+str(j)
@@ -151,7 +151,7 @@ connection = MongoClient()
 db = connection.timeit_test
 noddy = db.noddy
 
-for i in xrange(10000):
+for i in range(10000):
     example = {'fields': {}}
     for j in range(20):
         example['fields']["key"+str(j)] = "value "+str(j)
@@ -181,7 +181,7 @@ class Noddy(Document):
 """
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)
@@ -197,7 +197,7 @@ myNoddys = Noddy.objects()
     print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     fields = {}
     for j in range(20):
@@ -215,7 +215,7 @@ myNoddys = Noddy.objects()
     print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)
@@ -231,7 +231,7 @@ myNoddys = Noddy.objects()
     print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)
@@ -247,7 +247,7 @@ myNoddys = Noddy.objects()
     print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)
@@ -263,7 +263,7 @@ myNoddys = Noddy.objects()
     print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)

From 5ae588833b7ba1498a95e2efda4f788d4297997b Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Wed, 25 Jun 2014 18:22:39 +0300
Subject: [PATCH 49/73] Allowed to switch databases for a specific query.

---
 mongoengine/context_managers.py  |  7 -------
 mongoengine/queryset/base.py     | 24 ++++++++++++++++++++----
 mongoengine/queryset/queryset.py |  7 +++++++
 tests/queryset/queryset.py       | 16 ++++++++++++++++
 4 files changed, 43 insertions(+), 11 deletions(-)

diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py
index 13ed1009..cc860066 100644
--- a/mongoengine/context_managers.py
+++ b/mongoengine/context_managers.py
@@ -1,6 +1,5 @@
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
-from mongoengine.queryset import QuerySet
 
 
 __all__ = ("switch_db", "switch_collection", "no_dereference",
@@ -162,12 +161,6 @@ class no_sub_classes(object):
         return self.cls
 
 
-class QuerySetNoDeRef(QuerySet):
-    """Special no_dereference QuerySet"""
-    def __dereference(items, max_depth=1, instance=None, name=None):
-            return items
-
-
 class query_counter(object):
     """ Query_counter context manager to get the number of queries. """
 
diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index c2ad027e..a8d204b3 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -13,11 +13,11 @@ import pymongo
 from pymongo.common import validate_read_preference
 
 from mongoengine import signals
+from mongoengine.context_managers import switch_db
 from mongoengine.common import _import_class
 from mongoengine.base.common import get_document
 from mongoengine.errors import (OperationError, NotUniqueError,
                                 InvalidQueryError, LookUpError)
-
 from mongoengine.queryset import transform
 from mongoengine.queryset.field_list import QueryFieldList
 from mongoengine.queryset.visitor import Q, QNode
@@ -389,7 +389,7 @@ class BaseQuerySet(object):
                 ref_q = document_cls.objects(**{field_name + '__in': self})
                 ref_q_count = ref_q.count()
                 if (doc != document_cls and ref_q_count > 0
-                   or (doc == document_cls and ref_q_count > 0)):
+                    or (doc == document_cls and ref_q_count > 0)):
                     ref_q.delete(write_concern=write_concern)
             elif rule == NULLIFY:
                 document_cls.objects(**{field_name + '__in': self}).update(
@@ -522,6 +522,19 @@ class BaseQuerySet(object):
 
         return self
 
+    def using(self, alias):
+        """This method is for controlling which database the QuerySet will be evaluated against if you are using more than one database.
+
+        :param alias: The database alias
+
+        .. versionadded:: 0.8
+        """
+
+        with switch_db(self._document, alias) as cls:
+            collection = cls._get_collection()
+
+        return self.clone_into(self.__class__(self._document, collection))
+
     def clone(self):
         """Creates a copy of the current
           :class:`~mongoengine.queryset.QuerySet`
@@ -926,7 +939,7 @@ class BaseQuerySet(object):
             mr_args['out'] = output
 
         results = getattr(queryset._collection, map_reduce_function)(
-                          map_f, reduce_f, **mr_args)
+            map_f, reduce_f, **mr_args)
 
         if map_reduce_function == 'map_reduce':
             results = results.find()
@@ -1362,7 +1375,7 @@ class BaseQuerySet(object):
                 for subdoc in subclasses:
                     try:
                         subfield = ".".join(f.db_field for f in
-                                        subdoc._lookup_field(field.split('.')))
+                                            subdoc._lookup_field(field.split('.')))
                         ret.append(subfield)
                         found = True
                         break
@@ -1450,6 +1463,7 @@ class BaseQuerySet(object):
                     # type of this field and use the corresponding
                     # .to_python(...)
                     from mongoengine.fields import EmbeddedDocumentField
+
                     obj = self._document
                     for chunk in path.split('.'):
                         obj = getattr(obj, chunk, None)
@@ -1460,6 +1474,7 @@ class BaseQuerySet(object):
                     if obj and data is not None:
                         data = obj.to_python(data)
             return data
+
         return clean(row)
 
     def _sub_js_fields(self, code):
@@ -1468,6 +1483,7 @@ class BaseQuerySet(object):
         substituted for the MongoDB name of the field (specified using the
         :attr:`name` keyword argument in a field's constructor).
         """
+
         def field_sub(match):
             # Extract just the field name, and look up the field objects
             field_name = match.group(1).split('.')
diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py
index 1437e76b..cebfcc50 100644
--- a/mongoengine/queryset/queryset.py
+++ b/mongoengine/queryset/queryset.py
@@ -155,3 +155,10 @@ class QuerySetNoCache(BaseQuerySet):
             queryset = self.clone()
         queryset.rewind()
         return queryset
+
+
+class QuerySetNoDeRef(QuerySet):
+    """Special no_dereference QuerySet"""
+
+    def __dereference(items, max_depth=1, instance=None, name=None):
+        return items
\ No newline at end of file
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 7ff2965d..25ede817 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -29,6 +29,7 @@ class QuerySetTest(unittest.TestCase):
 
     def setUp(self):
         connect(db='mongoenginetest')
+        connect(db='mongoenginetest2', alias='test2')
 
         class PersonMeta(EmbeddedDocument):
             weight = IntField()
@@ -2957,6 +2958,21 @@ class QuerySetTest(unittest.TestCase):
 
         Number.drop_collection()
 
+    def test_using(self):
+        """Ensure that switching databases for a queryset is possible
+        """
+        class Number2(Document):
+            n = IntField()
+
+        Number2.drop_collection()
+
+        for i in xrange(1, 10):
+            t = Number2(n=i)
+            t.switch_db('test2')
+            t.save()
+
+        self.assertEqual(len(Number2.objects.using('test2')), 9)
+
     def test_unset_reference(self):
         class Comment(Document):
             text = StringField()

From 67a65a2aa9ecd77de372511aa54541100923d3d3 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Thu, 26 Jun 2014 11:17:57 +0300
Subject: [PATCH 50/73] Installing unittest2 on Python 2.6.

---
 setup.py                 | 3 +++
 tests/test_connection.py | 7 ++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 85707d00..f8f0d6da 100644
--- a/setup.py
+++ b/setup.py
@@ -58,6 +58,9 @@ if sys.version_info[0] == 3:
 else:
     extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil']
 
+    if sys.version_info[0] == 2 and sys.version_info[1] == 6:
+        extra_opts['tests_require'].append('unittest2')
+
 setup(name='mongoengine',
       version=VERSION,
       author='Harry Marr',
diff --git a/tests/test_connection.py b/tests/test_connection.py
index bf615ceb..a5b1b089 100644
--- a/tests/test_connection.py
+++ b/tests/test_connection.py
@@ -1,6 +1,11 @@
 import sys
 sys.path[0:0] = [""]
-import unittest
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
 import datetime
 
 import pymongo

From cae91ce0c564089b418d0fd508d80c7f923df949 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Thu, 26 Jun 2014 12:31:07 +0300
Subject: [PATCH 51/73] Convert codebase to Python 3 using 2to3 before running
 benchmarks.

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index fff35ca9..6281e6b4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -58,6 +58,7 @@ install:
     - python setup.py install
 script:
     - python setup.py test
+    - 2to3 . -w
     - python benchmark.py
 notifications:
   irc: "irc.freenode.org#mongoengine"

From 0e7878b406bc23b46140edd54912e4491595fac4 Mon Sep 17 00:00:00 2001
From: Omer Katz <omer.drow@gamil.com>
Date: Thu, 26 Jun 2014 12:41:26 +0300
Subject: [PATCH 52/73] Only run 2to3 on Python 3.x. Makes sense no?

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 6281e6b4..5f10339b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -58,7 +58,7 @@ install:
     - python setup.py install
 script:
     - python setup.py test
-    - 2to3 . -w
+    - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then 2to3 . -w; fi;
     - python benchmark.py
 notifications:
   irc: "irc.freenode.org#mongoengine"

From b02a31d4b920981f6b7330de23e78518f334cc0d Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 14:44:44 +0100
Subject: [PATCH 53/73] Updated .travis.yml

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5739909b..c7397fd3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,8 +8,7 @@ python:
     - "3.3"
 env:
   - PYMONGO=dev DJANGO=1.6
-  - PYMONGO=dev DJANGO=1.5.5
-  - PYMONGO=dev DJANGO=1.4.10
+  - PYMONGO=dev DJANGO=1.5.8
   - PYMONGO=2.5 DJANGO=1.6
   - PYMONGO=2.5 DJANGO=1.5.5
   - PYMONGO=2.5 DJANGO=1.4.10
@@ -31,3 +30,4 @@ notifications:
 branches:
   only:
     - master
+    - "0.8"

From dd51589f67fd02f3049aadea5268a79cd3799092 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 16:02:40 +0100
Subject: [PATCH 54/73] Updates

---
 setup.py                   |  2 +-
 tests/document/instance.py |  4 ++--
 tests/queryset/queryset.py | 12 ++++++------
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/setup.py b/setup.py
index 85707d00..25075275 100644
--- a/setup.py
+++ b/setup.py
@@ -72,7 +72,7 @@ setup(name='mongoengine',
       long_description=LONG_DESCRIPTION,
       platforms=['any'],
       classifiers=CLASSIFIERS,
-      install_requires=['pymongo>=2.5'],
+      install_requires=['pymongo>=2.7'],
       test_suite='nose.collector',
       **extra_opts
 )
diff --git a/tests/document/instance.py b/tests/document/instance.py
index afb27e0d..6f04ac1d 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -57,7 +57,7 @@ class InstanceTest(unittest.TestCase):
             date = DateTimeField(default=datetime.now)
             meta = {
                 'max_documents': 10,
-                'max_size': 90000,
+                'max_size': 4096,
             }
 
         Log.drop_collection()
@@ -75,7 +75,7 @@ class InstanceTest(unittest.TestCase):
         options = Log.objects._collection.options()
         self.assertEqual(options['capped'], True)
         self.assertEqual(options['max'], 10)
-        self.assertEqual(options['size'], 90000)
+        self.assertTrue(options['size'] >= 4096)
 
         # Check that the document cannot be redefined with different options
         def recreate_log_document():
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index a2438e21..688b1bce 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -650,7 +650,7 @@ class QuerySetTest(unittest.TestCase):
                 blogs.append(Blog(title="post %s" % i, posts=[post1, post2]))
 
             Blog.objects.insert(blogs, load_bulk=False)
-            self.assertEqual(q, 1)  # 1 for the insert
+            self.assertEqual(q, 99)  # profiling logs each doc now :(
 
         Blog.drop_collection()
         Blog.ensure_indexes()
@@ -659,7 +659,7 @@ class QuerySetTest(unittest.TestCase):
             self.assertEqual(q, 0)
 
             Blog.objects.insert(blogs)
-            self.assertEqual(q, 2)  # 1 for insert, and 1 for in bulk fetch
+            self.assertEqual(q, 100)  # 99 or insert, and 1 for in bulk fetch
 
         Blog.drop_collection()
 
@@ -3943,7 +3943,7 @@ class QuerySetTest(unittest.TestCase):
             if qs:
                 pass
 
-            op = q.db.system.profile.find({"ns": 
+            op = q.db.system.profile.find({"ns":
                 {"$ne": "%s.system.indexes" % q.db.name}})[0]
 
             self.assertFalse('$orderby' in op['query'],
@@ -3969,7 +3969,7 @@ class QuerySetTest(unittest.TestCase):
             }
 
         Person.drop_collection()
-        
+
         Person(name="B").save()
         Person(name="C").save()
         Person(name="A").save()
@@ -3979,13 +3979,13 @@ class QuerySetTest(unittest.TestCase):
             if Person.objects:
                 pass
 
-            op = q.db.system.profile.find({"ns": 
+            op = q.db.system.profile.find({"ns":
                 {"$ne": "%s.system.indexes" % q.db.name}})[0]
 
             self.assertFalse('$orderby' in op['query'],
                 'BaseQuerySet must remove orderby from meta in boolen test')
 
-            self.assertEqual(Person.objects.first().name, 'A') 
+            self.assertEqual(Person.objects.first().name, 'A')
             self.assertTrue(Person.objects._has_data(),
                             'Cursor has data and returned False')
 

From 5b6c8c191f1418471d8b9f616d5fbded4a797238 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 14:44:44 +0100
Subject: [PATCH 55/73] Updated .travis.yml

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5739909b..3d9d6611 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,8 +8,7 @@ python:
     - "3.3"
 env:
   - PYMONGO=dev DJANGO=1.6
-  - PYMONGO=dev DJANGO=1.5.5
-  - PYMONGO=dev DJANGO=1.4.10
+  - PYMONGO=dev DJANGO=1.5.8
   - PYMONGO=2.5 DJANGO=1.6
   - PYMONGO=2.5 DJANGO=1.5.5
   - PYMONGO=2.5 DJANGO=1.4.10
@@ -31,3 +30,4 @@ notifications:
 branches:
   only:
     - master
+    - "0.9"

From eb9003187d4ce70bdbb32709fc6cb9f20ce7eb16 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 16:13:01 +0100
Subject: [PATCH 56/73] Updated changelog & authors #673

---
 AUTHORS            | 1 +
 docs/changelog.rst | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/AUTHORS b/AUTHORS
index 170a00e5..c86df67c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -191,3 +191,4 @@ that much better:
  * Damien Churchill (https://github.com/damoxc)
  * Jonathan Simon Prates (https://github.com/jonathansp)
  * Thiago Papageorgiou (https://github.com/tmpapageorgiou)
+ * Omer Katz (https://github.com/thedrow)
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 51134238..b94388a1 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -2,6 +2,11 @@
 Changelog
 =========
 
+
+Changes in 0.9.X - DEV
+======================
+- pypy support #673
+
 Changes in 0.8.7
 ================
 - Calling reload on deleted / nonexistant documents raises DoesNotExist (#538)

From 11724aa5555ad7f87e0208b3ce01ac8b7f31323c Mon Sep 17 00:00:00 2001
From: Dmitry Konishchev <konishchev@gmail.com>
Date: Thu, 26 Jun 2014 16:18:42 +0100
Subject: [PATCH 57/73] QuerySet.modify() method to provide find_and_modify()
 like behaviour

---
 docs/guide/querying.rst      |   5 +-
 mongoengine/queryset/base.py |  54 +++++++++++++++++++
 tests/queryset/__init__.py   |   1 +
 tests/queryset/modify.py     | 102 +++++++++++++++++++++++++++++++++++
 4 files changed, 160 insertions(+), 2 deletions(-)
 create mode 100644 tests/queryset/modify.py

diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst
index 32cbb94e..96beea5f 100644
--- a/docs/guide/querying.rst
+++ b/docs/guide/querying.rst
@@ -488,8 +488,9 @@ calling it with keyword arguments::
 Atomic updates
 ==============
 Documents may be updated atomically by using the
-:meth:`~mongoengine.queryset.QuerySet.update_one` and
-:meth:`~mongoengine.queryset.QuerySet.update` methods on a
+:meth:`~mongoengine.queryset.QuerySet.update_one`,
+:meth:`~mongoengine.queryset.QuerySet.update` and
+:meth:`~mongoengine.queryset.QuerySet.modify` methods on a
 :meth:`~mongoengine.queryset.QuerySet`. There are several different "modifiers"
 that you may use with these methods:
 
diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 89a5e5fb..db60deba 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -10,6 +10,7 @@ import warnings
 from bson.code import Code
 from bson import json_util
 import pymongo
+import pymongo.errors
 from pymongo.common import validate_read_preference
 
 from mongoengine import signals
@@ -484,6 +485,59 @@ class BaseQuerySet(object):
         return self.update(
             upsert=upsert, multi=False, write_concern=write_concern, **update)
 
+    def modify(self, upsert=False, full_response=False, remove=False, new=False, **update):
+        """Update and return the updated document.
+
+        Returns either the document before or after modification based on `new`
+        parameter. If no documents match the query and `upsert` is false,
+        returns ``None``. If upserting and `new` is false, returns ``None``.
+
+        If the full_response parameter is ``True``, the return value will be
+        the entire response object from the server, including the 'ok' and
+        'lastErrorObject' fields, rather than just the modified document.
+        This is useful mainly because the 'lastErrorObject' document holds
+        information about the command's execution.
+
+        :param upsert: insert if document doesn't exist (default ``False``)
+        :param full_response: return the entire response object from the
+            server (default ``False``)
+        :param remove: remove rather than updating (default ``False``)
+        :param new: return updated rather than original document
+            (default ``False``)
+        :param update: Django-style update keyword arguments
+
+        .. versionadded:: 0.9
+        """
+
+        if remove and new:
+            raise OperationError("Conflicting parameters: remove and new")
+
+        if not update and not upsert and not remove:
+            raise OperationError("No update parameters, must either update or remove")
+
+        queryset = self.clone()
+        query = queryset._query
+        update = transform.update(queryset._document, **update)
+        sort = queryset._ordering
+
+        try:
+            result = queryset._collection.find_and_modify(
+                query, update, upsert=upsert, sort=sort, remove=remove, new=new,
+                full_response=full_response, **self._cursor_args)
+        except pymongo.errors.DuplicateKeyError, err:
+            raise NotUniqueError(u"Update failed (%s)" % err)
+        except pymongo.errors.OperationFailure, err:
+            raise OperationError(u"Update failed (%s)" % err)
+
+        if full_response:
+            if result["value"] is not None:
+                result["value"] = self._document._from_son(result["value"])
+        else:
+            if result is not None:
+                result = self._document._from_son(result)
+
+        return result
+
     def with_id(self, object_id):
         """Retrieve the object matching the id provided.  Uses `object_id` only
         and raises InvalidQueryError if a filter has been applied. Returns
diff --git a/tests/queryset/__init__.py b/tests/queryset/__init__.py
index 8a93c19f..c36b2684 100644
--- a/tests/queryset/__init__.py
+++ b/tests/queryset/__init__.py
@@ -3,3 +3,4 @@ from field_list import *
 from queryset import *
 from visitor import *
 from geo import *
+from modify import *
\ No newline at end of file
diff --git a/tests/queryset/modify.py b/tests/queryset/modify.py
new file mode 100644
index 00000000..e0c7d1fe
--- /dev/null
+++ b/tests/queryset/modify.py
@@ -0,0 +1,102 @@
+import sys
+sys.path[0:0] = [""]
+
+import unittest
+
+from mongoengine import connect, Document, IntField
+
+__all__ = ("FindAndModifyTest",)
+
+
+class Doc(Document):
+    id = IntField(primary_key=True)
+    value = IntField()
+
+
+class FindAndModifyTest(unittest.TestCase):
+
+    def setUp(self):
+        connect(db="mongoenginetest")
+        Doc.drop_collection()
+
+    def assertDbEqual(self, docs):
+        self.assertEqual(list(Doc._collection.find().sort("id")), docs)
+
+    def test_modify(self):
+        Doc(id=0, value=0).save()
+        doc = Doc(id=1, value=1).save()
+
+        old_doc = Doc.objects(id=1).modify(set__value=-1)
+        self.assertEqual(old_doc.to_json(), doc.to_json())
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+
+    def test_modify_with_new(self):
+        Doc(id=0, value=0).save()
+        doc = Doc(id=1, value=1).save()
+
+        new_doc = Doc.objects(id=1).modify(set__value=-1, new=True)
+        doc.value = -1
+        self.assertEqual(new_doc.to_json(), doc.to_json())
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+
+    def test_modify_not_existing(self):
+        Doc(id=0, value=0).save()
+        self.assertEqual(Doc.objects(id=1).modify(set__value=-1), None)
+        self.assertDbEqual([{"_id": 0, "value": 0}])
+
+    def test_modify_with_upsert(self):
+        Doc(id=0, value=0).save()
+        old_doc = Doc.objects(id=1).modify(set__value=1, upsert=True)
+        self.assertEqual(old_doc, None)
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
+
+    def test_modify_with_upsert_existing(self):
+        Doc(id=0, value=0).save()
+        doc = Doc(id=1, value=1).save()
+
+        old_doc = Doc.objects(id=1).modify(set__value=-1, upsert=True)
+        self.assertEqual(old_doc.to_json(), doc.to_json())
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+
+    def test_modify_with_upsert_with_new(self):
+        Doc(id=0, value=0).save()
+        new_doc = Doc.objects(id=1).modify(upsert=True, new=True, set__value=1)
+        self.assertEqual(new_doc.to_mongo(), {"_id": 1, "value": 1})
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
+
+    def test_modify_with_remove(self):
+        Doc(id=0, value=0).save()
+        doc = Doc(id=1, value=1).save()
+
+        old_doc = Doc.objects(id=1).modify(remove=True)
+        self.assertEqual(old_doc.to_json(), doc.to_json())
+        self.assertDbEqual([{"_id": 0, "value": 0}])
+
+    def test_find_and_modify_with_remove_not_existing(self):
+        Doc(id=0, value=0).save()
+        self.assertEqual(Doc.objects(id=1).modify(remove=True), None)
+        self.assertDbEqual([{"_id": 0, "value": 0}])
+
+    def test_modify_with_order_by(self):
+        Doc(id=0, value=3).save()
+        Doc(id=1, value=2).save()
+        Doc(id=2, value=1).save()
+        doc = Doc(id=3, value=0).save()
+
+        old_doc = Doc.objects().order_by("-id").modify(set__value=-1)
+        self.assertEqual(old_doc.to_json(), doc.to_json())
+        self.assertDbEqual([
+            {"_id": 0, "value": 3}, {"_id": 1, "value": 2},
+            {"_id": 2, "value": 1}, {"_id": 3, "value": -1}])
+
+    def test_modify_with_fields(self):
+        Doc(id=0, value=0).save()
+        Doc(id=1, value=1).save()
+
+        old_doc = Doc.objects(id=1).only("id").modify(set__value=-1)
+        self.assertEqual(old_doc.to_mongo(), {"_id": 1})
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

From 2c07d7736847fa1157ab38c6247ecaa0ac34c0e1 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 16:24:37 +0100
Subject: [PATCH 58/73] Updated changelog

Enabled connection pooling
---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index b94388a1..bead491c 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,7 @@ Changelog
 Changes in 0.9.X - DEV
 ======================
 - pypy support #673
+- Enabled connection pooling #674
 
 Changes in 0.8.7
 ================

From d1d59722774f59d3ee9b935bfda9ff7e373b43c3 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 16:34:02 +0100
Subject: [PATCH 59/73] Removed support for old versions

- Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x.
- Removing support for Python < 2.6.6
---
 .travis.yml                     | 34 +--------------------------------
 docs/changelog.rst              |  2 ++
 mongoengine/base/document.py    |  7 +------
 mongoengine/python_support.py   | 32 -------------------------------
 mongoengine/queryset/visitor.py |  5 +++--
 setup.py                        |  6 ++++--
 6 files changed, 11 insertions(+), 75 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c20e52af..40736165 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,44 +11,12 @@ python:
 env:
   - PYMONGO=dev DJANGO=1.6.5
   - PYMONGO=dev DJANGO=1.5.8
-  - PYMONGO=dev DJANGO=1.4.13
-  - PYMONGO=2.5.2 DJANGO=1.6.5
-  - PYMONGO=2.5.2 DJANGO=1.5.8
-  - PYMONGO=2.5.2 DJANGO=1.4.13
-  - PYMONGO=2.6.3 DJANGO=1.6.5
-  - PYMONGO=2.6.3 DJANGO=1.5.8
-  - PYMONGO=2.6.3 DJANGO=1.4.13
   - PYMONGO=2.7.1 DJANGO=1.6.5
   - PYMONGO=2.7.1 DJANGO=1.5.8
-  - PYMONGO=2.7.1 DJANGO=1.4.13
 
 matrix:
     fast_finish: true
-    exclude:
-        - python: "3.2"
-          env: PYMONGO=dev DJANGO=1.4.13
-        - python: "3.2"
-          env: PYMONGO=2.5.2 DJANGO=1.4.13
-        - python: "3.2"
-          env: PYMONGO=2.6.3 DJANGO=1.4.13
-        - python: "3.2"
-          env: PYMONGO=2.7.1 DJANGO=1.4.13
-        - python: "3.3"
-          env: PYMONGO=dev DJANGO=1.4.13
-        - python: "3.3"
-          env: PYMONGO=2.5.2 DJANGO=1.4.13
-        - python: "3.3"
-          env: PYMONGO=2.6.3 DJANGO=1.4.13
-        - python: "3.3"
-          env: PYMONGO=2.7.1 DJANGO=1.4.13
-        - python: "3.4"
-          env: PYMONGO=dev DJANGO=1.4.13
-        - python: "3.4"
-          env: PYMONGO=2.5.2 DJANGO=1.4.13
-        - python: "3.4"
-          env: PYMONGO=2.6.3 DJANGO=1.4.13
-        - python: "3.4"
-          env: PYMONGO=2.7.1 DJANGO=1.4.13
+
 install:
     - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk
     - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi
diff --git a/docs/changelog.rst b/docs/changelog.rst
index bead491c..c980e904 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -7,6 +7,8 @@ Changes in 0.9.X - DEV
 ======================
 - pypy support #673
 - Enabled connection pooling #674
+- Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x.
+- Removing support for Python < 2.6.6
 
 Changes in 0.8.7
 ================
diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index f5eae8ff..43b865ce 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -13,8 +13,7 @@ from mongoengine import signals
 from mongoengine.common import _import_class
 from mongoengine.errors import (ValidationError, InvalidDocumentError,
                                 LookUpError)
-from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type,
-                                        to_str_keys_recursive)
+from mongoengine.python_support import PY3, txt_type
 
 from mongoengine.base.common import get_document, ALLOW_INHERITANCE
 from mongoengine.base.datastructures import BaseDict, BaseList
@@ -545,10 +544,6 @@ class BaseDocument(object):
         # class if unavailable
         class_name = son.get('_cls', cls._class_name)
         data = dict(("%s" % key, value) for key, value in son.iteritems())
-        if not UNICODE_KWARGS:
-            # python 2.6.4 and lower cannot handle unicode keys
-            # passed to class constructor example: cls(**data)
-            to_str_keys_recursive(data)
 
         # Return correct subclass for document type
         if class_name != cls._class_name:
diff --git a/mongoengine/python_support.py b/mongoengine/python_support.py
index 097740eb..2c4df00c 100644
--- a/mongoengine/python_support.py
+++ b/mongoengine/python_support.py
@@ -3,8 +3,6 @@
 import sys
 
 PY3 = sys.version_info[0] == 3
-PY25 = sys.version_info[:2] == (2, 5)
-UNICODE_KWARGS = int(''.join([str(x) for x in sys.version_info[:3]])) > 264
 
 if PY3:
     import codecs
@@ -29,33 +27,3 @@ else:
     txt_type = unicode
 
 str_types = (bin_type, txt_type)
-
-if PY25:
-    def product(*args, **kwds):
-        pools = map(tuple, args) * kwds.get('repeat', 1)
-        result = [[]]
-        for pool in pools:
-            result = [x + [y] for x in result for y in pool]
-        for prod in result:
-            yield tuple(prod)
-    reduce = reduce
-else:
-    from itertools import product
-    from functools import reduce
-
-
-# For use with Python 2.5
-# converts all keys from unicode to str for d and all nested dictionaries
-def to_str_keys_recursive(d):
-    if isinstance(d, list):
-        for val in d:
-            if isinstance(val, (dict, list)):
-                to_str_keys_recursive(val)
-    elif isinstance(d, dict):
-        for key, val in d.items():
-            if isinstance(val, (dict, list)):
-                to_str_keys_recursive(val)
-            if isinstance(key, unicode):
-                d[str(key)] = d.pop(key)
-    else:
-        raise ValueError("non list/dict parameter not allowed")
diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py
index 41f4ebf8..a39b05f0 100644
--- a/mongoengine/queryset/visitor.py
+++ b/mongoengine/queryset/visitor.py
@@ -1,8 +1,9 @@
 import copy
 
-from mongoengine.errors import InvalidQueryError
-from mongoengine.python_support import product, reduce
+from itertools import product
+from functools import reduce
 
+from mongoengine.errors import InvalidQueryError
 from mongoengine.queryset import transform
 
 __all__ = ('Q',)
diff --git a/setup.py b/setup.py
index d37cede1..7270331a 100644
--- a/setup.py
+++ b/setup.py
@@ -38,12 +38,14 @@ CLASSIFIERS = [
     'Operating System :: OS Independent',
     'Programming Language :: Python',
     "Programming Language :: Python :: 2",
-    "Programming Language :: Python :: 2.6",
+    "Programming Language :: Python :: 2.6.6",
     "Programming Language :: Python :: 2.7",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.1",
     "Programming Language :: Python :: 3.2",
+    "Programming Language :: Python :: 3.3",
+    "Programming Language :: Python :: 3.4",
     "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
     'Topic :: Database',
     'Topic :: Software Development :: Libraries :: Python Modules',
 ]

From da0a1bbe9f63059bd745d378e723c184c079c90c Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 17:13:21 +0100
Subject: [PATCH 60/73] Fix test_using

---
 tests/queryset/queryset.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 649a3582..b97f9519 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -16,7 +16,7 @@ from bson import ObjectId
 from mongoengine import *
 from mongoengine.connection import get_connection
 from mongoengine.python_support import PY3
-from mongoengine.context_managers import query_counter
+from mongoengine.context_managers import query_counter, switch_db
 from mongoengine.queryset import (QuerySet, QuerySetManager,
                                   MultipleObjectsReturned, DoesNotExist,
                                   queryset_manager)
@@ -3035,8 +3035,10 @@ class QuerySetTest(unittest.TestCase):
             n = IntField()
 
         Number2.drop_collection()
+        with switch_db(Number2, 'test2') as Number2:
+            Number2.drop_collection()
 
-        for i in xrange(1, 10):
+        for i in range(1, 10):
             t = Number2(n=i)
             t.switch_db('test2')
             t.save()

From cfc31eead324e71958c2e69b0117b515a0ae6bc2 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 17:13:35 +0100
Subject: [PATCH 61/73] Fixed $maxDistance location for geoJSON $near queries
 with MongoDB 2.6+

Closes #664
---
 mongoengine/queryset/transform.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index 27e41ad2..8e88e9fe 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -3,6 +3,7 @@ from collections import defaultdict
 import pymongo
 from bson import SON
 
+from mongoengine.connection import get_connection
 from mongoengine.common import _import_class
 from mongoengine.errors import InvalidQueryError, LookUpError
 
@@ -115,14 +116,21 @@ def query(_doc_cls=None, _field_operation=False, **query):
             if key in mongo_query and isinstance(mongo_query[key], dict):
                 mongo_query[key].update(value)
                 # $maxDistance needs to come last - convert to SON
-                if '$maxDistance' in mongo_query[key]:
-                    value_dict = mongo_query[key]
+                value_dict = mongo_query[key]
+                if ('$maxDistance' in value_dict and '$near' in value_dict and
+                    isinstance(value_dict['$near'], dict)):
+
                     value_son = SON()
                     for k, v in value_dict.iteritems():
                         if k == '$maxDistance':
                             continue
                         value_son[k] = v
-                    value_son['$maxDistance'] = value_dict['$maxDistance']
+                    if (get_connection().max_wire_version <= 1):
+                        value_son['$maxDistance'] = value_dict['$maxDistance']
+                    else:
+                        value_son['$near'] = SON(value_son['$near'])
+                        value_son['$near']['$maxDistance'] = value_dict['$maxDistance']
+
                     mongo_query[key] = value_son
             else:
                 # Store for manually merging later

From 70651ce99499d354a93c4776548eb48cc748523a Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 19:24:52 +0100
Subject: [PATCH 62/73] Fix as_pymongo bug

---
 mongoengine/queryset/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 5fd84195..4c37d989 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -147,7 +147,7 @@ class BaseQuerySet(object):
                     queryset._document._from_son(queryset._cursor[key],
                                                  _auto_dereference=self._auto_dereference))
             if queryset._as_pymongo:
-                return queryset._get_as_pymongo(queryset._cursor.next())
+                return queryset._get_as_pymongo(queryset._cursor[key])
             return queryset._document._from_son(queryset._cursor[key],
                                                 _auto_dereference=self._auto_dereference)
         raise AttributeError

From 4ee212e7d58a3fa39b5a4c18a311453b262a422c Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 19:25:05 +0100
Subject: [PATCH 63/73] Skip Test due to server bug in 2.6

---
 tests/queryset/geo.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py
index 65ab519a..5148a48e 100644
--- a/tests/queryset/geo.py
+++ b/tests/queryset/geo.py
@@ -5,6 +5,8 @@ import unittest
 from datetime import datetime, timedelta
 from mongoengine import *
 
+from nose.plugins.skip import SkipTest
+
 __all__ = ("GeoQueriesTest",)
 
 
@@ -139,6 +141,7 @@ class GeoQueriesTest(unittest.TestCase):
     def test_spherical_geospatial_operators(self):
         """Ensure that spherical geospatial queries are working
         """
+        raise SkipTest("https://jira.mongodb.org/browse/SERVER-14039")
         class Point(Document):
             location = GeoPointField()
 

From 3a0c69005ba274ec6d039b44c17af1c78fd48223 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 19:41:40 +0100
Subject: [PATCH 64/73] Update AUTHORS and Changelog

Refs: #664, #677, #676, #673, #674, #655, #657, #626, #625, #619, #613, #608, #511, #559
---
 AUTHORS            |  5 +++++
 docs/changelog.rst | 17 +++++++++++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index c86df67c..82daf0d8 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -192,3 +192,8 @@ that much better:
  * Jonathan Simon Prates (https://github.com/jonathansp)
  * Thiago Papageorgiou (https://github.com/tmpapageorgiou)
  * Omer Katz (https://github.com/thedrow)
+ * Falcon Dai (https://github.com/falcondai)
+ * Polyrabbit (https://github.com/polyrabbit)
+ * Sagiv Malihi (https://github.com/sagivmalihi)
+ * Dmitry Konishchev (https://github.com/KonishchevDmitry)
+ *
\ No newline at end of file
diff --git a/docs/changelog.rst b/docs/changelog.rst
index c980e904..88959617 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -5,10 +5,23 @@ Changelog
 
 Changes in 0.9.X - DEV
 ======================
-- pypy support #673
-- Enabled connection pooling #674
+
 - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x.
 - Removing support for Python < 2.6.6
+- Fixed $maxDistance location for geoJSON $near queries with MongoDB 2.6+ #664
+- QuerySet.modify() method to provide find_and_modify() like behaviour #677
+- Added support for the using() method on a queryset #676
+- PYPY support #673
+- Connection pooling #674
+- Avoid to open all documents from cursors in an if stmt #655
+- Ability to clear the ordering #657
+- Raise NotUniqueError in Document.update() on pymongo.errors.DuplicateKeyError #626
+- Slots - memory improvements #625
+- Fixed incorrectly split a query key when it ends with "_" #619
+- Geo docs updates #613
+- Workaround a dateutil bug #608
+- Conditional save for atomic-style operations #511
+- Allow dynamic dictionary-style field access #559
 
 Changes in 0.8.7
 ================

From c0a5b16a7f0d7f29be972f653e6d351e58be0b78 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Thu, 26 Jun 2014 19:52:05 +0100
Subject: [PATCH 65/73] Travis bump

---
 AUTHORS | 1 -
 1 file changed, 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index 82daf0d8..2f832a3d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -196,4 +196,3 @@ that much better:
  * Polyrabbit (https://github.com/polyrabbit)
  * Sagiv Malihi (https://github.com/sagivmalihi)
  * Dmitry Konishchev (https://github.com/KonishchevDmitry)
- *
\ No newline at end of file

From 5be5685a090b86b90609037257457824ab62a467 Mon Sep 17 00:00:00 2001
From: Martyn Smith <martyn@dollyfish.net.nz>
Date: Fri, 27 Jun 2014 09:06:17 +0100
Subject: [PATCH 66/73] Test to illustrate failure in changed attribute
 tracking

---
 tests/document/delta.py | 42 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/tests/document/delta.py b/tests/document/delta.py
index b0f5f01a..738dfa78 100644
--- a/tests/document/delta.py
+++ b/tests/document/delta.py
@@ -735,5 +735,47 @@ class DeltaTest(unittest.TestCase):
         mydoc._clear_changed_fields()
         self.assertEqual([], mydoc._get_changed_fields())
 
+    def test_referenced_object_changed_attributes(self):
+        """Ensures that when you save a new reference to a field, the referenced object isn't altered"""
+
+        class Organization(Document):
+            name = StringField()
+
+        class User(Document):
+            name = StringField()
+            org = ReferenceField('Organization', required=True)
+
+        Organization.drop_collection()
+        User.drop_collection()
+
+        org1 = Organization(name='Org 1')
+        org1.save()
+
+        org2 = Organization(name='Org 2')
+        org2.save()
+
+        user = User(name='Fred', org=org1)
+        user.save()
+
+        org1.reload()
+        org2.reload()
+        user.reload()
+        self.assertEqual(org1.name, 'Org 1')
+        self.assertEqual(org2.name, 'Org 2')
+        self.assertEqual(user.name, 'Fred')
+
+        user.name = 'Harold'
+        user.org = org2
+
+        org2.name = 'New Org 2'
+        self.assertEqual(org2.name, 'New Org 2')
+
+        user.save()
+        org2.save()
+
+        self.assertEqual(org2.name, 'New Org 2')
+        org2.reload()
+        self.assertEqual(org2.name, 'New Org 2')
+
 if __name__ == '__main__':
     unittest.main()

From cd63865d311956f82396a5415f1f65f7d17cd654 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Fri, 27 Jun 2014 09:08:07 +0100
Subject: [PATCH 67/73] Fix clear_changed_fields() clearing unsaved documents
 bug #602

---
 AUTHORS                      |  1 +
 docs/changelog.rst           |  1 +
 mongoengine/base/document.py | 12 +++++++-----
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 2f832a3d..a0801b3c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -196,3 +196,4 @@ that much better:
  * Polyrabbit (https://github.com/polyrabbit)
  * Sagiv Malihi (https://github.com/sagivmalihi)
  * Dmitry Konishchev (https://github.com/KonishchevDmitry)
+ * Martyn Smith (https://github.com/martynsmith)
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 88959617..5c3ac1c1 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,7 @@ Changelog
 Changes in 0.9.X - DEV
 ======================
 
+- Fix clear_changed_fields() clearing unsaved documents bug #602
 - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x.
 - Removing support for Python < 2.6.6
 - Fixed $maxDistance location for geoJSON $near queries with MongoDB 2.6+ #664
diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index 08401823..f6eec628 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -16,7 +16,7 @@ from mongoengine.errors import (ValidationError, InvalidDocumentError,
 from mongoengine.python_support import PY3, txt_type
 
 from mongoengine.base.common import get_document, ALLOW_INHERITANCE
-from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict, SemiStrictDict 
+from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict, SemiStrictDict
 from mongoengine.base.fields import ComplexBaseField
 
 __all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
@@ -54,12 +54,12 @@ class BaseDocument(object):
                 values[name] = value
         __auto_convert = values.pop("__auto_convert", True)
         signals.pre_init.send(self.__class__, document=self, values=values)
-        
+
         if self.STRICT and not self._dynamic:
             self._data = StrictDict.create(allowed_keys=self._fields.keys())()
         else:
             self._data = SemiStrictDict.create(allowed_keys=self._fields.keys())()
-            
+
         self._dynamic_fields = SON()
 
         # Assign default values to instance
@@ -150,7 +150,7 @@ class BaseDocument(object):
         try:
             self__initialised = self._initialised
         except AttributeError:
-            self__initialised = False 
+            self__initialised = False
         # Check if the user has created a new instance of a class
         if (self._is_document and self__initialised
            and self__created and name == self._meta['id_field']):
@@ -407,6 +407,8 @@ class BaseDocument(object):
                 else:
                     data = getattr(data, part, None)
                 if hasattr(data, "_changed_fields"):
+                    if hasattr(data, "_is_document") and data._is_document:
+                        continue
                     data._changed_fields = []
         self._changed_fields = []
 
@@ -596,7 +598,7 @@ class BaseDocument(object):
             msg = ("Invalid data to create a `%s` instance.\n%s"
                    % (cls._class_name, errors))
             raise InvalidDocumentError(msg)
-        
+
         if cls.STRICT:
             data = dict((k, v) for k,v in data.iteritems() if k in cls._fields)
         obj = cls(__auto_convert=False, **data)

From 72a051f2d3d4b09d19d1b0a47f35deff434ffc05 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Fri, 27 Jun 2014 09:12:05 +0100
Subject: [PATCH 68/73] Update AUTHORS & Changelog #557

---
 AUTHORS            | 1 +
 docs/changelog.rst | 1 +
 2 files changed, 2 insertions(+)

diff --git a/AUTHORS b/AUTHORS
index a0801b3c..339a58a3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -197,3 +197,4 @@ that much better:
  * Sagiv Malihi (https://github.com/sagivmalihi)
  * Dmitry Konishchev (https://github.com/KonishchevDmitry)
  * Martyn Smith (https://github.com/martynsmith)
+ * Andrei Zbikowski (https://github.com/b1naryth1ef)
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 5c3ac1c1..08fb24ef 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,7 @@ Changelog
 Changes in 0.9.X - DEV
 ======================
 
+- Fixes issue with recursive embedded document errors #557
 - Fix clear_changed_fields() clearing unsaved documents bug #602
 - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x.
 - Removing support for Python < 2.6.6

From 7f36ea55f57da2f159f5c15b08ada234bcffe38c Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Fri, 27 Jun 2014 09:14:56 +0100
Subject: [PATCH 69/73] Fix bulk test where behaviour changes based on mongo
 version

---
 tests/queryset/queryset.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index b97f9519..5b18dc63 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -651,7 +651,10 @@ class QuerySetTest(unittest.TestCase):
                 blogs.append(Blog(title="post %s" % i, posts=[post1, post2]))
 
             Blog.objects.insert(blogs, load_bulk=False)
-            self.assertEqual(q, 99)  # profiling logs each doc now :(
+            if (get_connection().max_wire_version <= 1):
+                self.assertEqual(q, 1)
+            else:
+                self.assertEqual(q, 99)  # profiling logs each doc now in the bulk op
 
         Blog.drop_collection()
         Blog.ensure_indexes()
@@ -660,7 +663,10 @@ class QuerySetTest(unittest.TestCase):
             self.assertEqual(q, 0)
 
             Blog.objects.insert(blogs)
-            self.assertEqual(q, 100)  # 99 or insert, and 1 for in bulk fetch
+            if (get_connection().max_wire_version <= 1):
+                self.assertEqual(q, 2) # 1 for insert, and 1 for in bulk fetch
+            else:
+                self.assertEqual(q, 100)  # 99 for insert, and 1 for in bulk fetch
 
         Blog.drop_collection()
 

From 0971ad0a80c24a7661fd7ce6a173379fb07d1b62 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Fri, 27 Jun 2014 09:31:01 +0100
Subject: [PATCH 70/73] Update changelog & authors - #636

---
 AUTHORS            | 1 +
 docs/changelog.rst | 1 +
 2 files changed, 2 insertions(+)

diff --git a/AUTHORS b/AUTHORS
index 339a58a3..b9ab34df 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -198,3 +198,4 @@ that much better:
  * Dmitry Konishchev (https://github.com/KonishchevDmitry)
  * Martyn Smith (https://github.com/martynsmith)
  * Andrei Zbikowski (https://github.com/b1naryth1ef)
+ * Ronald van Rij (https://github.com/ronaldvanrij)
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 08fb24ef..2ed0001b 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,7 @@ Changelog
 Changes in 0.9.X - DEV
 ======================
 
+- Fix id shard key save issue #636
 - Fixes issue with recursive embedded document errors #557
 - Fix clear_changed_fields() clearing unsaved documents bug #602
 - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x.

From 5e776a07dd2a267017a3d91723ea7dbf1ab826a4 Mon Sep 17 00:00:00 2001
From: Stefan Wojcik <wojcikstefan@gmail.com>
Date: Fri, 7 Mar 2014 16:08:06 -0800
Subject: [PATCH 71/73] allow ordering to be cleared

---
 mongoengine/queryset/base.py |  2 +-
 tests/queryset/queryset.py   | 98 +++++++++++++++---------------------
 2 files changed, 42 insertions(+), 58 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 4c37d989..be5d66b0 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -192,7 +192,7 @@ class BaseQuerySet(object):
         .. versionadded:: 0.3
         """
         queryset = self.clone()
-        queryset = queryset.limit(2)
+        queryset = queryset.order_by().limit(2)
         queryset = queryset.filter(*q_objs, **query)
 
         try:
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 5b18dc63..4770c3e6 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -14,8 +14,8 @@ from pymongo.read_preferences import ReadPreference
 from bson import ObjectId
 
 from mongoengine import *
-from mongoengine.connection import get_connection
 from mongoengine.python_support import PY3
+from mongoengine.connection import get_connection
 from mongoengine.context_managers import query_counter, switch_db
 from mongoengine.queryset import (QuerySet, QuerySetManager,
                                   MultipleObjectsReturned, DoesNotExist,
@@ -25,6 +25,12 @@ from mongoengine.errors import InvalidQueryError
 __all__ = ("QuerySetTest",)
 
 
+class db_ops_tracker(query_counter):
+    def get_ops(self):
+        ignore_query = {"ns": {"$ne": "%s.system.indexes" % self.db.name}}
+        return list(self.db.system.profile.find(ignore_query))
+
+
 class QuerySetTest(unittest.TestCase):
 
     def setUp(self):
@@ -1048,74 +1054,52 @@ class QuerySetTest(unittest.TestCase):
         self.assertSequence(qs, expected)
 
     def test_clear_ordering(self):
-        """ Make sure one can clear the query set ordering by applying a
-        consecutive order_by()
+        """ Ensure that the default ordering can be cleared by calling order_by().
         """
+        class BlogPost(Document):
+            title = StringField()
+            published_date = DateTimeField()
 
-        class Person(Document):
-            name = StringField()
-
-        Person.drop_collection()
-        Person(name="A").save()
-        Person(name="B").save()
-
-        qs = Person.objects.order_by('-name')
-
-        # Make sure we can clear a previously specified ordering
-        with query_counter() as q:
-            lst = list(qs.order_by())
-
-            op = q.db.system.profile.find({"ns":
-                {"$ne": "%s.system.indexes" % q.db.name}})[0]
-
-            self.assertTrue('$orderby' not in op['query'])
-            self.assertEqual(lst[0].name, 'A')
-
-        # Make sure previously specified ordering is preserved during
-        # consecutive calls to the same query set
-        with query_counter() as q:
-            lst = list(qs)
-
-            op = q.db.system.profile.find({"ns":
-                {"$ne": "%s.system.indexes" % q.db.name}})[0]
-
-            self.assertTrue('$orderby' in op['query'])
-            self.assertEqual(lst[0].name, 'B')
-
-    def test_clear_default_ordering(self):
-
-        class Person(Document):
-            name = StringField()
             meta = {
-                'ordering': ['-name']
+                'ordering': ['-published_date']
             }
 
-        Person.drop_collection()
-        Person(name="A").save()
-        Person(name="B").save()
+        BlogPost.drop_collection()
 
-        qs = Person.objects
+        with db_ops_tracker() as q:
+            BlogPost.objects.filter(title='whatever').first()
+            self.assertEqual(len(q.get_ops()), 1)
+            self.assertEqual(q.get_ops()[0]['query']['$orderby'], {u'published_date': -1})
 
-        # Make sure clearing default ordering works
-        with query_counter() as q:
-            lst = list(qs.order_by())
+        with db_ops_tracker() as q:
+            BlogPost.objects.filter(title='whatever').order_by().first()
+            self.assertEqual(len(q.get_ops()), 1)
+            print q.get_ops()[0]['query']
+            self.assertFalse('$orderby' in q.get_ops()[0]['query'])
 
-            op = q.db.system.profile.find({"ns":
-                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+    def test_no_ordering_for_get(self):
+        """ Ensure that Doc.objects.get doesn't use any ordering.
+        """
+        class BlogPost(Document):
+            title = StringField()
+            published_date = DateTimeField()
 
-            self.assertTrue('$orderby' not in op['query'])
-            self.assertEqual(lst[0].name, 'A')
+            meta = {
+                'ordering': ['-published_date']
+            }
 
-        # Make sure default ordering is preserved during consecutive calls
-        # to the same query set
-        with query_counter() as q:
-            lst = list(qs)
+        BlogPost.objects.create(title='whatever', published_date=datetime.utcnow())
 
-            op = q.db.system.profile.find({"ns":
-                {"$ne": "%s.system.indexes" % q.db.name}})[0]
+        with db_ops_tracker() as q:
+            BlogPost.objects.get(title='whatever')
+            self.assertEqual(len(q.get_ops()), 1)
+            self.assertFalse('$orderby' in q.get_ops()[0]['query'])
 
-            self.assertTrue('$orderby' in op['query'])
-            self.assertEqual(lst[0].name, 'B')
+        # Ordering should be ignored for .get even if we set it explicitly
+        with db_ops_tracker() as q:
+            BlogPost.objects.order_by('-title').get(title='whatever')
+            self.assertEqual(len(q.get_ops()), 1)
+            self.assertFalse('$orderby' in q.get_ops()[0]['query'])
 
     def test_find_embedded(self):
         """Ensure that an embedded document is properly returned from a query.

From 2e4fb86b866fa0c9fe63c499a25b2f51935d9db3 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Fri, 27 Jun 2014 10:00:16 +0100
Subject: [PATCH 72/73] Don't query with $orderby for qs.get() #600

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 2ed0001b..45f7e020 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,7 @@ Changelog
 Changes in 0.9.X - DEV
 ======================
 
+- Don't query with $orderby for qs.get() #600
 - Fix id shard key save issue #636
 - Fixes issue with recursive embedded document errors #557
 - Fix clear_changed_fields() clearing unsaved documents bug #602

From 7013033ae415be314c6e7de91b6ea2bd321e06d7 Mon Sep 17 00:00:00 2001
From: Ross Lawley <ross.lawley@gmail.com>
Date: Fri, 27 Jun 2014 10:03:35 +0100
Subject: [PATCH 73/73] Update changelog & AUTHORS #594 #589

---
 AUTHORS            | 1 +
 docs/changelog.rst | 1 +
 2 files changed, 2 insertions(+)

diff --git a/AUTHORS b/AUTHORS
index b9ab34df..c6c47d79 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -199,3 +199,4 @@ that much better:
  * Martyn Smith (https://github.com/martynsmith)
  * Andrei Zbikowski (https://github.com/b1naryth1ef)
  * Ronald van Rij (https://github.com/ronaldvanrij)
+ * François Schmidts (https://github.com/jaesivsm)
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 45f7e020..c722b592 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,7 @@ Changelog
 Changes in 0.9.X - DEV
 ======================
 
+- post_save signal now has access to delta information about field changes #594 #589
 - Don't query with $orderby for qs.get() #600
 - Fix id shard key save issue #636
 - Fixes issue with recursive embedded document errors #557