From b9e922c658f6275fc8446bf1ffdedf688f708f25 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Mon, 12 Jun 2017 04:50:13 +0000
Subject: [PATCH 001/121] support multiple operator #1510

---
 AUTHORS                    |  1 +
 mongoengine/base/common.py |  7 ++++---
 tests/queryset/queryset.py | 14 ++++++++++++++
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 1d724718..88d4bbe1 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -243,3 +243,4 @@ that much better:
  * Victor Varvaryuk
  * Stanislav Kaledin (https://github.com/sallyruthstruik)
  * Dmitry Yantsen (https://github.com/mrTable)
+ * Erdenezul Batmunkh (https://github.com/erdenezul)
diff --git a/mongoengine/base/common.py b/mongoengine/base/common.py
index b9971ff7..f80471ef 100644
--- a/mongoengine/base/common.py
+++ b/mongoengine/base/common.py
@@ -3,9 +3,10 @@ from mongoengine.errors import NotRegistered
 __all__ = ('UPDATE_OPERATORS', 'get_document', '_document_registry')
 
 
-UPDATE_OPERATORS = set(['set', 'unset', 'inc', 'dec', 'pop', 'push',
-                        'push_all', 'pull', 'pull_all', 'add_to_set',
-                        'set_on_insert', 'min', 'max', 'rename'])
+UPDATE_OPERATORS = set(['set', 'unset', 'inc', 'dec', 'mul',
+                        'pop', 'push', 'push_all', 'pull',
+                        'pull_all', 'add_to_set', 'set_on_insert',
+                        'min', 'max', 'rename'])
 
 
 _document_registry = {}
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index d97b307d..7666d030 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -589,6 +589,20 @@ class QuerySetTest(unittest.TestCase):
         Scores.objects(id=scores.id).update(max__high_score=500)
         self.assertEqual(Scores.objects.get(id=scores.id).high_score, 1000)
 
+    @needs_mongodb_v26
+    def test_update_multiple(self):
+        class Product(Document):
+            item = StringField()
+            price = FloatField()
+
+        product = Product.objects.create(item='ABC', price=10.99)
+        product = Product.objects.create(item='ABC', price=10.99)
+        Product.objects(id=product.id).update(mul__price=1.25)
+        self.assertEqual(Product.objects.get(id=product.id).price, 13.7375)
+        unknown_product = Product.objects.create(item='Unknown')
+        Product.objects(id=unknown_product.id).update(mul__price=100)
+        self.assertEqual(Product.objects.get(id=unknown_product.id).price, 0)
+
     def test_updates_can_have_match_operators(self):
 
         class Comment(EmbeddedDocument):

From 6903eed4e770bcb605df948728121ce75149d0fa Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Thu, 15 Jun 2017 06:08:40 +0000
Subject: [PATCH 002/121] support position in 'push' #1565

---
 mongoengine/queryset/transform.py | 16 +++++++++++++---
 tests/queryset/queryset.py        | 23 +++++++++++++++++++++++
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index bb04ee37..2c1b7fdc 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -284,9 +284,11 @@ def update(_doc_cls=None, **update):
             if isinstance(field, GeoJsonBaseField):
                 value = field.to_mongo(value)
 
-            if op in (None, 'set', 'push', 'pull'):
+            if op == 'push' and isinstance(value, (list, tuple, set)):
+                value = [field.prepare_query_value(op, v) for v in value]
+            elif op in (None, 'set', 'push', 'pull'):
                 if field.required or value is not None:
-                    value = field.prepare_query_value(op, value)
+                        value = field.prepare_query_value(op, value)
             elif op in ('pushAll', 'pullAll'):
                 value = [field.prepare_query_value(op, v) for v in value]
             elif op in ('addToSet', 'setOnInsert'):
@@ -302,6 +304,10 @@ def update(_doc_cls=None, **update):
             value = {match: value}
 
         key = '.'.join(parts)
+        position = None
+        if parts[-1].isdigit() and isinstance(value, (list, tuple, set)):
+            key = parts[0]
+            position = int(parts[-1])
 
         if not op:
             raise InvalidQueryError('Updates must supply an operation '
@@ -333,10 +339,14 @@ def update(_doc_cls=None, **update):
                 value = {key: value}
         elif op == 'addToSet' and isinstance(value, list):
             value = {key: {'$each': value}}
+        elif op == 'push' and isinstance(value, list):
+            if position is not None:
+                value = {key: {'$each': value, '$position': position}}
+            else:
+                value = {key: {'$each': value}}
         else:
             value = {key: value}
         key = '$' + op
-
         if key not in mongo_update:
             mongo_update[key] = value
         elif key in mongo_update and isinstance(mongo_update[key], dict):
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index d97b307d..9341a214 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1903,6 +1903,29 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    def test_update_push_with_position(self):
+        """Ensure that the 'push' update with position works properly.
+        """
+        class BlogPost(Document):
+            slug = StringField()
+            tags = ListField(StringField())
+
+        BlogPost.drop_collection()
+
+        post = BlogPost(slug="test")
+        post.save()
+
+        BlogPost.objects.filter(id=post.id).update(push__tags="code")
+        BlogPost.objects.filter(id=post.id).update(push__tags__0=["mongodb", "python"])
+        post.reload()
+        self.assertEqual(post.tags[0], "mongodb")
+        self.assertEqual(post.tags[1], "python")
+        self.assertEqual(post.tags[2], "code")
+
+        BlogPost.objects.filter(id=post.id).update(set__tags__2="java")
+        post.reload()
+        self.assertEqual(post.tags[2], "java")
+
     def test_update_push_and_pull_add_to_set(self):
         """Ensure that the 'pull' update operation works correctly.
         """

From f63ad2dd69c40d87962fe05137e618c09418ed5e Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Thu, 15 Jun 2017 07:36:14 +0000
Subject: [PATCH 003/121] dont test in mongoDB v2.4 #1565

---
 tests/queryset/queryset.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 9341a214..7be3f8d7 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1903,6 +1903,7 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    @needs_mongodb_v26
     def test_update_push_with_position(self):
         """Ensure that the 'push' update with position works properly.
         """

From f3ee4a5dac9a6bcccdd11f9be3afc702e344e8a2 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Mon, 19 Jun 2017 02:59:17 +0000
Subject: [PATCH 004/121] add tests for push operator #1565

---
 AUTHORS                    |  1 +
 tests/document/instance.py | 41 ++++++++++++++++++++++++++++++++++++++
 tests/queryset/modify.py   | 25 ++++++++++++++++++++++-
 3 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index 1d724718..88d4bbe1 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -243,3 +243,4 @@ that much better:
  * Victor Varvaryuk
  * Stanislav Kaledin (https://github.com/sallyruthstruik)
  * Dmitry Yantsen (https://github.com/mrTable)
+ * Erdenezul Batmunkh (https://github.com/erdenezul)
diff --git a/tests/document/instance.py b/tests/document/instance.py
index 37bbe337..1456f0a1 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -22,6 +22,8 @@ from mongoengine.queryset import NULLIFY, Q
 from mongoengine.context_managers import switch_db, query_counter
 from mongoengine import signals
 
+from tests.utils import needs_mongodb_v26
+
 TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__),
                                '../fields/mongoengine.png')
 
@@ -775,6 +777,7 @@ class InstanceTest(unittest.TestCase):
 
         self.assertDbEqual([dict(doc.to_mongo())])
 
+
     def test_modify_invalid_query(self):
         doc1 = self.Person(name="bob", age=10).save()
         doc2 = self.Person(name="jim", age=20).save()
@@ -826,6 +829,27 @@ class InstanceTest(unittest.TestCase):
 
         self.assertDbEqual([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
 
+    @needs_mongodb_v26
+    def test_modity_push_position(self):
+        class BlogPost(Document):
+            slug = StringField()
+            tags = ListField(StringField())
+
+        other_blog = BlogPost(slug="ABC", tags=["code", "java", "python"]).save()
+
+        blog = BlogPost(slug="ABC", tags=["python"]).save()
+        blog_copy = blog._from_son(blog.to_mongo())
+
+        assert blog.modify(push__tags__0=["code", "java"])
+        blog_copy.tags = ["code", "java", "python"]
+        assert blog.to_json() == blog_copy.to_json()
+        assert blog._get_changed_fields() == []
+
+        docs = [dict(other_blog.to_mongo()), dict(blog.to_mongo())]
+        self.assertEqual(
+            list(BlogPost._get_collection().find().sort("id")),
+            sorted(docs, key=lambda doc: doc["_id"]))
+
     def test_save(self):
         """Ensure that a document may be saved in the database."""
 
@@ -3149,6 +3173,23 @@ class InstanceTest(unittest.TestCase):
 
         person.update(set__height=2.0)
 
+    @needs_mongodb_v26
+    def test_push_with_position(self):
+        """Ensure that push with position works properly for an instance."""
+        class BlogPost(Document):
+            slug = StringField()
+            tags = ListField(StringField())
+
+        blog = BlogPost()
+        blog.slug = "ABC"
+        blog.tags = ["python"]
+        blog.save()
+
+        blog.update(push__tags__0=["mongodb", "code"])
+        blog.reload()
+        self.assertEqual(blog.tags[0], "mongodb")
+        self.assertEqual(blog.tags[2], "python")
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/queryset/modify.py b/tests/queryset/modify.py
index 607937f6..4bda9718 100644
--- a/tests/queryset/modify.py
+++ b/tests/queryset/modify.py
@@ -1,6 +1,6 @@
 import unittest
 
-from mongoengine import connect, Document, IntField
+from mongoengine import connect, Document, IntField, StringField, ListField
 
 __all__ = ("FindAndModifyTest",)
 
@@ -94,6 +94,29 @@ class FindAndModifyTest(unittest.TestCase):
         self.assertEqual(old_doc.to_mongo(), {"_id": 1})
         self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
+    def test_modify_with_push(self):
+        class BlogPost(Document):
+            id = StringField(primary_key=True)
+            tags = ListField(StringField())
+
+        BlogPost.drop_collection()
+
+        BlogPost(id="ABC").save()
+        BlogPost(id="BCD").save()
+        blog = BlogPost.objects(id="ABC").modify(push__tags="code")
+
+        self.assertEqual(blog.to_mongo(), {"_id": "ABC", "tags": []})
+        docs = [{"_id": "ABC", "tags":["code"]}, {"_id": "BCD", "tags":[]}]
+        self.assertEqual(list(BlogPost._collection.find().sort("id")), docs)
+
+        another_blog = BlogPost.objects(id="BCD").modify(push__tags="java")
+        self.assertEqual(another_blog.to_mongo(), {"_id": "BCD", "tags": []})
+        another_blog = BlogPost.objects(id="BCD").modify(push__tags__0=["python"])
+        self.assertEqual(another_blog.to_mongo(), {"_id": "BCD", "tags": ["java"]})
+        docs = [{"_id": "ABC", "tags":["code"]},
+                {"_id": "BCD", "tags":["python", "java"]}]
+        self.assertEqual(list(BlogPost._collection.find().sort("id")), docs)
+
 
 if __name__ == '__main__':
     unittest.main()

From 7782aa7379306c8223edd33aeec6b47f2f145b83 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Mon, 19 Jun 2017 03:11:59 +0000
Subject: [PATCH 005/121] do not test position push in mongodb_v2.4 #1565

---
 tests/document/instance.py | 1 -
 tests/queryset/modify.py   | 3 +++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index 1456f0a1..26ab0d97 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -777,7 +777,6 @@ class InstanceTest(unittest.TestCase):
 
         self.assertDbEqual([dict(doc.to_mongo())])
 
-
     def test_modify_invalid_query(self):
         doc1 = self.Person(name="bob", age=10).save()
         doc2 = self.Person(name="jim", age=20).save()
diff --git a/tests/queryset/modify.py b/tests/queryset/modify.py
index 4bda9718..44bdc3ff 100644
--- a/tests/queryset/modify.py
+++ b/tests/queryset/modify.py
@@ -2,6 +2,8 @@ import unittest
 
 from mongoengine import connect, Document, IntField, StringField, ListField
 
+from tests.utils import needs_mongodb_v26
+
 __all__ = ("FindAndModifyTest",)
 
 
@@ -94,6 +96,7 @@ class FindAndModifyTest(unittest.TestCase):
         self.assertEqual(old_doc.to_mongo(), {"_id": 1})
         self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
+    @needs_mongodb_v26
     def test_modify_with_push(self):
         class BlogPost(Document):
             id = StringField(primary_key=True)

From fb00b79d1945a05c08f642aa35efbba5e92558a8 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Mon, 19 Jun 2017 03:28:34 +0000
Subject: [PATCH 006/121] add docs for positional push operator #1565

---
 docs/guide/querying.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst
index 0bb19658..f1594dd2 100644
--- a/docs/guide/querying.rst
+++ b/docs/guide/querying.rst
@@ -565,6 +565,15 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`::
     >>> post.tags
     ['database', 'mongodb']
 
+From MongoDB version 2.6, push operator supports $position value which allows
+to push values with index.
+    >>> post = BlogPost(title="Test", tags=["mongo"])
+    >>> post.save()
+    >>> post.update(push__tags__0=["database", "code"])
+    >>> post.reload()
+    >>> post.tags
+    ['database', 'code', 'mongo']
+
 .. note::
     Currently only top level lists are handled, future versions of mongodb /
     pymongo plan to support nested positional operators.  See `The $ positional

From 71c3c632d7efb673aad2f1e79e564537df202e7b Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Mon, 19 Jun 2017 06:00:21 +0000
Subject: [PATCH 007/121] add test case for reverse_delete_rule with pull #1519

---
 tests/document/instance.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index 37bbe337..f9f4285e 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -1866,6 +1866,25 @@ class InstanceTest(unittest.TestCase):
         author.delete()
         self.assertEqual(BlogPost.objects.count(), 0)
 
+    def test_reverse_delete_rule_pull(self):
+        """Ensure that a referenced document is also deleted with
+        pull.
+        """
+        class Record(Document):
+            name = StringField()
+            children = ListField(ReferenceField('self', reverse_delete_rule=PULL))
+
+        Record.drop_collection()
+
+        parent_record = Record(name='parent').save()
+        child_record = Record(name='child').save()
+        parent_record.children.append(child_record)
+        parent_record.save()
+
+        child_record.delete()
+        self.assertEqual(Record.objects(name='parent').get().children, [])
+
+
     def test_reverse_delete_rule_with_custom_id_field(self):
         """Ensure that a referenced document with custom primary key
         is also deleted upon deletion.

From a7cab513695ba1814d9316719429d63249fb97dc Mon Sep 17 00:00:00 2001
From: Davidrjx <1058960881@qq.com>
Date: Thu, 13 Jul 2017 18:07:36 +0800
Subject: [PATCH 008/121] Use a set literal in _clean_settings (#1585)

---
 AUTHORS                   |  1 +
 mongoengine/connection.py | 11 ++++++-----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 1d724718..96a7850e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -243,3 +243,4 @@ that much better:
  * Victor Varvaryuk
  * Stanislav Kaledin (https://github.com/sallyruthstruik)
  * Dmitry Yantsen (https://github.com/mrTable)
+ * Renjianxin (https://github.com/Davidrjx)
diff --git a/mongoengine/connection.py b/mongoengine/connection.py
index 7eae810f..34ff4dc3 100644
--- a/mongoengine/connection.py
+++ b/mongoengine/connection.py
@@ -146,13 +146,14 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
         raise MongoEngineConnectionError(msg)
 
     def _clean_settings(settings_dict):
-        irrelevant_fields = set([
-            'name', 'username', 'password', 'authentication_source',
-            'authentication_mechanism'
-        ])
+        # set literal more efficient than calling set function
+        irrelevant_fields_set = {
+            'name', 'username', 'password',
+            'authentication_source', 'authentication_mechanism'
+        }
         return {
             k: v for k, v in settings_dict.items()
-            if k not in irrelevant_fields
+            if k not in irrelevant_fields_set
         }
 
     # Retrieve a copy of the connection settings associated with the requested

From 3dcc9bc1433e550a3151ade5f31efef7e39f2be3 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezulbatmunkh@Erdenezuls-iMac.local>
Date: Thu, 13 Jul 2017 22:59:21 +0800
Subject: [PATCH 009/121] use explicit tests and fix unneccessary indent #1565

---
 mongoengine/queryset/transform.py |  2 +-
 tests/document/instance.py        | 26 ++++++++++----------------
 tests/queryset/queryset.py        |  6 ++----
 3 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index 2c1b7fdc..b9c8b130 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -288,7 +288,7 @@ def update(_doc_cls=None, **update):
                 value = [field.prepare_query_value(op, v) for v in value]
             elif op in (None, 'set', 'push', 'pull'):
                 if field.required or value is not None:
-                        value = field.prepare_query_value(op, value)
+                    value = field.prepare_query_value(op, value)
             elif op in ('pushAll', 'pullAll'):
                 value = [field.prepare_query_value(op, v) for v in value]
             elif op in ('addToSet', 'setOnInsert'):
diff --git a/tests/document/instance.py b/tests/document/instance.py
index 26ab0d97..609bc900 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -829,25 +829,20 @@ class InstanceTest(unittest.TestCase):
         self.assertDbEqual([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
 
     @needs_mongodb_v26
-    def test_modity_push_position(self):
+    def test_modify_with_positional_push(self):
         class BlogPost(Document):
-            slug = StringField()
             tags = ListField(StringField())
 
-        other_blog = BlogPost(slug="ABC", tags=["code", "java", "python"]).save()
+        post = BlogPost.objects.create(tags=['python'])
+        self.assertEqual(post.tags, ['python'])
+        post.modify(push__tags__0=['code', 'mongo'])
+        self.assertEqual(post.tags, ['code', 'mongo', 'python'])
 
-        blog = BlogPost(slug="ABC", tags=["python"]).save()
-        blog_copy = blog._from_son(blog.to_mongo())
-
-        assert blog.modify(push__tags__0=["code", "java"])
-        blog_copy.tags = ["code", "java", "python"]
-        assert blog.to_json() == blog_copy.to_json()
-        assert blog._get_changed_fields() == []
-
-        docs = [dict(other_blog.to_mongo()), dict(blog.to_mongo())]
+        # Assert same order of the list items is maintained in the db
         self.assertEqual(
-            list(BlogPost._get_collection().find().sort("id")),
-            sorted(docs, key=lambda doc: doc["_id"]))
+            BlogPost._get_collection().find_one({'_id': post.pk})['tags'],
+            ['code', 'mongo', 'python']
+        )
 
     def test_save(self):
         """Ensure that a document may be saved in the database."""
@@ -3186,8 +3181,7 @@ class InstanceTest(unittest.TestCase):
 
         blog.update(push__tags__0=["mongodb", "code"])
         blog.reload()
-        self.assertEqual(blog.tags[0], "mongodb")
-        self.assertEqual(blog.tags[2], "python")
+        self.assertEqual(blog.tags, ['mongodb', 'code', 'python'])
 
 
 if __name__ == '__main__':
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 7be3f8d7..f4e4670d 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1919,13 +1919,11 @@ class QuerySetTest(unittest.TestCase):
         BlogPost.objects.filter(id=post.id).update(push__tags="code")
         BlogPost.objects.filter(id=post.id).update(push__tags__0=["mongodb", "python"])
         post.reload()
-        self.assertEqual(post.tags[0], "mongodb")
-        self.assertEqual(post.tags[1], "python")
-        self.assertEqual(post.tags[2], "code")
+        self.assertEqual(post.tags, ['mongodb', 'python', 'code'])
 
         BlogPost.objects.filter(id=post.id).update(set__tags__2="java")
         post.reload()
-        self.assertEqual(post.tags[2], "java")
+        self.assertEqual(post.tags, ['mongodb', 'python', 'java'])
 
     def test_update_push_and_pull_add_to_set(self):
         """Ensure that the 'pull' update operation works correctly.

From 433f10ef93cbfa2d2f0a8cb704e8333704ebf362 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Mon, 31 Jul 2017 05:15:23 +0000
Subject: [PATCH 010/121] position support singular value #1565

---
 mongoengine/queryset/transform.py | 17 ++++++++++-------
 tests/queryset/queryset.py        |  5 +++++
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index b9c8b130..b01d3d41 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -304,10 +304,6 @@ def update(_doc_cls=None, **update):
             value = {match: value}
 
         key = '.'.join(parts)
-        position = None
-        if parts[-1].isdigit() and isinstance(value, (list, tuple, set)):
-            key = parts[0]
-            position = int(parts[-1])
 
         if not op:
             raise InvalidQueryError('Updates must supply an operation '
@@ -339,11 +335,18 @@ def update(_doc_cls=None, **update):
                 value = {key: value}
         elif op == 'addToSet' and isinstance(value, list):
             value = {key: {'$each': value}}
-        elif op == 'push' and isinstance(value, list):
-            if position is not None:
+        elif op == 'push':
+            if parts[-1].isdigit() and op == 'push':
+                key = parts[0]
+                position = int(parts[-1])
+                # position modifier must appear with each.
+                if not isinstance(value, (set, tuple, list)):
+                    value = [value]
                 value = {key: {'$each': value, '$position': position}}
-            else:
+            elif isinstance(value, list):
                 value = {key: {'$each': value}}
+            else:
+                value = {key: value}
         else:
             value = {key: value}
         key = '$' + op
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index f4e4670d..fe97b765 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1925,6 +1925,11 @@ class QuerySetTest(unittest.TestCase):
         post.reload()
         self.assertEqual(post.tags, ['mongodb', 'python', 'java'])
 
+        #test push with singular value
+        BlogPost.objects.filter(id=post.id).update(push__tags__0='scala')
+        post.reload()
+        self.assertEqual(post.tags, ['scala', 'mongodb', 'python', 'java'])
+
     def test_update_push_and_pull_add_to_set(self):
         """Ensure that the 'pull' update operation works correctly.
         """

From 34fca9d6f5cb8ea09af7dc63a480354c8ecbc06d Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezulbatmunkh@Erdenezuls-iMac.local>
Date: Mon, 31 Jul 2017 18:32:34 +0800
Subject: [PATCH 011/121] Add clear comment and tests for positional push #1565

---
 mongoengine/queryset/transform.py |  5 +++--
 tests/queryset/modify.py          | 33 +++++++++++++++++++------------
 tests/queryset/queryset.py        |  3 +--
 3 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index b01d3d41..a9907ada 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -336,10 +336,11 @@ def update(_doc_cls=None, **update):
         elif op == 'addToSet' and isinstance(value, list):
             value = {key: {'$each': value}}
         elif op == 'push':
-            if parts[-1].isdigit() and op == 'push':
+            if parts[-1].isdigit():
                 key = parts[0]
                 position = int(parts[-1])
-                # position modifier must appear with each.
+                # $position expects an iterable. If pushing a single value,
+                # wrap it in a list.
                 if not isinstance(value, (set, tuple, list)):
                     value = [value]
                 value = {key: {'$each': value, '$position': position}}
diff --git a/tests/queryset/modify.py b/tests/queryset/modify.py
index 44bdc3ff..fe410d19 100644
--- a/tests/queryset/modify.py
+++ b/tests/queryset/modify.py
@@ -99,26 +99,33 @@ class FindAndModifyTest(unittest.TestCase):
     @needs_mongodb_v26
     def test_modify_with_push(self):
         class BlogPost(Document):
-            id = StringField(primary_key=True)
             tags = ListField(StringField())
 
         BlogPost.drop_collection()
 
         BlogPost(id="ABC").save()
-        BlogPost(id="BCD").save()
-        blog = BlogPost.objects(id="ABC").modify(push__tags="code")
 
-        self.assertEqual(blog.to_mongo(), {"_id": "ABC", "tags": []})
-        docs = [{"_id": "ABC", "tags":["code"]}, {"_id": "BCD", "tags":[]}]
-        self.assertEqual(list(BlogPost._collection.find().sort("id")), docs)
+        # Push a new tag via modify with new=False (default).
+        blog = BlogPost(pk='ABC').modify(push__tags='code')
+        self.assertEqual(blog.tags, [])
+        blog.reload()
+        self.assertEqual(blog.tags, ['code'])
 
-        another_blog = BlogPost.objects(id="BCD").modify(push__tags="java")
-        self.assertEqual(another_blog.to_mongo(), {"_id": "BCD", "tags": []})
-        another_blog = BlogPost.objects(id="BCD").modify(push__tags__0=["python"])
-        self.assertEqual(another_blog.to_mongo(), {"_id": "BCD", "tags": ["java"]})
-        docs = [{"_id": "ABC", "tags":["code"]},
-                {"_id": "BCD", "tags":["python", "java"]}]
-        self.assertEqual(list(BlogPost._collection.find().sort("id")), docs)
+        # Push a new tag via modify with new=True.
+        blog = BlogPost.objects(pk='ABC').modify(push__tags='java', new=True)
+        self.assertEqual(blog.tags, ['code', 'java'])
+
+        # Push a new tag with a positional argument.
+        blog = BlogPost.objects(pk='ABC').modify(
+            push__tags__0='python',
+            new=True)
+        self.assertEqual(blog.tags, ['python', 'code', 'java'])
+
+        # Push multiple new tags with a positional argument.
+        blog = BlogPost.objects(pk='ABC').modify(
+            push__tags__1=['go', 'rust'],
+            new=True)
+        self.assertEqual(blog.tags, ['python', 'go', 'rust', 'code', 'java'])
 
 
 if __name__ == '__main__':
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index fe97b765..c78ed985 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1913,8 +1913,7 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
-        post = BlogPost(slug="test")
-        post.save()
+        post = BlogPost.objects.create(slug="test")
 
         BlogPost.objects.filter(id=post.id).update(push__tags="code")
         BlogPost.objects.filter(id=post.id).update(push__tags__0=["mongodb", "python"])

From f09256a24e83623b194792234f6a9e4c6944e2a6 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezulbatmunkh@Erdenezuls-iMac.local>
Date: Mon, 31 Jul 2017 18:49:52 +0800
Subject: [PATCH 012/121] Fix modify tests #1565

---
 tests/queryset/modify.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/queryset/modify.py b/tests/queryset/modify.py
index fe410d19..b37f9b73 100644
--- a/tests/queryset/modify.py
+++ b/tests/queryset/modify.py
@@ -103,26 +103,26 @@ class FindAndModifyTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
-        BlogPost(id="ABC").save()
+        blog = BlogPost.objects.create()
 
         # Push a new tag via modify with new=False (default).
-        blog = BlogPost(pk='ABC').modify(push__tags='code')
+        BlogPost(id=blog.id).modify(push__tags='code')
         self.assertEqual(blog.tags, [])
         blog.reload()
         self.assertEqual(blog.tags, ['code'])
 
         # Push a new tag via modify with new=True.
-        blog = BlogPost.objects(pk='ABC').modify(push__tags='java', new=True)
+        blog = BlogPost.objects(id=blog.id).modify(push__tags='java', new=True)
         self.assertEqual(blog.tags, ['code', 'java'])
 
         # Push a new tag with a positional argument.
-        blog = BlogPost.objects(pk='ABC').modify(
+        blog = BlogPost.objects(id=blog.id).modify(
             push__tags__0='python',
             new=True)
         self.assertEqual(blog.tags, ['python', 'code', 'java'])
 
         # Push multiple new tags with a positional argument.
-        blog = BlogPost.objects(pk='ABC').modify(
+        blog = BlogPost.objects(id=blog.id).modify(
             push__tags__1=['go', 'rust'],
             new=True)
         self.assertEqual(blog.tags, ['python', 'go', 'rust', 'code', 'java'])

From b9f3991d03fcd78ad5b81011d8420ab0287b29bd Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Fri, 18 Aug 2017 09:31:26 +0800
Subject: [PATCH 013/121] added as_pymongo test for PointField #1311

---
 tests/queryset/geo.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py
index 51a32382..38c0377e 100644
--- a/tests/queryset/geo.py
+++ b/tests/queryset/geo.py
@@ -510,6 +510,24 @@ class GeoQueriesTest(MongoDBTestCase):
         roads = Road.objects.filter(poly__geo_intersects={"$geometry": polygon}).count()
         self.assertEqual(1, roads)
 
+    def test_aspymongo_with_only(self):
+        """Ensure as_pymongo works with only"""
+        class Place(Document):
+            location = PointField()
+
+        Place.drop_collection()
+        p = Place(location=[24.946861267089844, 60.16311983618494])
+        p.save()
+        qs = Place.objects().only('location')
+        self.assertDictEqual(
+            qs.as_pymongo()[0]['location'],
+            {u'type': u'Point',
+             u'coordinates': [
+                24.946861267089844,
+                60.16311983618494]
+            }
+        )
+
     def test_2dsphere_point_sets_correctly(self):
         class Location(Document):
             loc = PointField()

From 21d1faa7938e4585fa12325037f0695e3572ee80 Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Thu, 24 Aug 2017 18:03:08 +0200
Subject: [PATCH 014/121] Fix .install_mongodb_on_travis.sh script (all credit
 to @erdenezul)

---
 .install_mongodb_on_travis.sh | 6 +++++-
 .travis.yml                   | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/.install_mongodb_on_travis.sh b/.install_mongodb_on_travis.sh
index 8563ae74..f2018411 100644
--- a/.install_mongodb_on_travis.sh
+++ b/.install_mongodb_on_travis.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 
+sudo apt-get remove mongodb-org-server
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
 
 if [ "$MONGODB" = "2.4" ]; then
@@ -13,7 +14,7 @@ elif [ "$MONGODB" = "2.6" ]; then
     sudo apt-get install mongodb-org-server=2.6.12
     # service should be started automatically
 elif [ "$MONGODB" = "3.0" ]; then
-    echo "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb.list
+    echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb.list
     sudo apt-get update
     sudo apt-get install mongodb-org-server=3.0.14
     # service should be started automatically
@@ -21,3 +22,6 @@ else
     echo "Invalid MongoDB version, expected 2.4, 2.6, or 3.0."
     exit 1
 fi;
+
+mkdir db
+1>db/logs mongod --dbpath=db &
diff --git a/.travis.yml b/.travis.yml
index 8f846ab1..5e90660d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,6 +42,8 @@ matrix:
 
 before_install:
 - bash .install_mongodb_on_travis.sh
+- sleep 15  # https://docs.travis-ci.com/user/database-setup/#MongoDB-does-not-immediately-accept-connections
+- mongo --eval 'db.version();'
 
 install:
 - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev

From 4f59c7f77fdda587386ec3a03b4de9caf1b0dadf Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Tue, 25 Jul 2017 16:26:07 +0200
Subject: [PATCH 015/121] Expose to user mongoengine.base.NON_FIELD_ERRORS

This variable is used to set the field containing the errors raised in a
clean function. Given those function are user-defined, users should be
able to get the name of the field to easily retreive their custom errors.
---
 mongoengine/base/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index 99c8af87..d05c7695 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -19,7 +19,7 @@ from mongoengine.common import _import_class
 from mongoengine.errors import (FieldDoesNotExist, InvalidDocumentError,
                                 LookUpError, OperationError, ValidationError)
 
-__all__ = ('BaseDocument',)
+__all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
 
 NON_FIELD_ERRORS = '__all__'
 

From bce859569f924fe051441b99c36d11795b5953a9 Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Tue, 25 Jul 2017 19:37:21 +0200
Subject: [PATCH 016/121] Remove SemiStrictDict to improve perfs

---
 docs/changelog.rst                 |  2 +-
 mongoengine/base/datastructures.py | 39 ----------------------------
 mongoengine/base/document.py       |  5 ++--
 tests/test_datastructures.py       | 41 +-----------------------------
 4 files changed, 4 insertions(+), 83 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index c60bbf09..1dcbfb51 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,7 +4,7 @@ Changelog
 
 Development
 ===========
-- (Fill this out as you fix issues and develop your features).
+- Improve performances by removing SemiStrictDict
 
 Changes in 0.14.0
 =================
diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py
index b9aca8fa..cd6ac641 100644
--- a/mongoengine/base/datastructures.py
+++ b/mongoengine/base/datastructures.py
@@ -445,42 +445,3 @@ class StrictDict(object):
 
             cls._classes[allowed_keys] = SpecificStrictDict
         return cls._classes[allowed_keys]
-
-
-class SemiStrictDict(StrictDict):
-    __slots__ = ('_extras', )
-    _classes = {}
-
-    def __getattr__(self, attr):
-        try:
-            super(SemiStrictDict, self).__getattr__(attr)
-        except AttributeError:
-            try:
-                return self.__getattribute__('_extras')[attr]
-            except KeyError as e:
-                raise AttributeError(e)
-
-    def __setattr__(self, attr, value):
-        try:
-            super(SemiStrictDict, self).__setattr__(attr, value)
-        except AttributeError:
-            try:
-                self._extras[attr] = value
-            except AttributeError:
-                self._extras = {attr: value}
-
-    def __delattr__(self, attr):
-        try:
-            super(SemiStrictDict, self).__delattr__(attr)
-        except AttributeError:
-            try:
-                del self._extras[attr]
-            except KeyError as e:
-                raise AttributeError(e)
-
-    def __iter__(self):
-        try:
-            extras_iter = iter(self.__getattribute__('_extras'))
-        except AttributeError:
-            extras_iter = ()
-        return itertools.chain(super(SemiStrictDict, self).__iter__(), extras_iter)
diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index d05c7695..f8ab73d0 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -13,7 +13,7 @@ from mongoengine import signals
 from mongoengine.base.common import get_document
 from mongoengine.base.datastructures import (BaseDict, BaseList,
                                              EmbeddedDocumentList,
-                                             SemiStrictDict, StrictDict)
+                                             StrictDict)
 from mongoengine.base.fields import ComplexBaseField
 from mongoengine.common import _import_class
 from mongoengine.errors import (FieldDoesNotExist, InvalidDocumentError,
@@ -79,8 +79,7 @@ class BaseDocument(object):
         if self.STRICT and not self._dynamic:
             self._data = StrictDict.create(allowed_keys=self._fields_ordered)()
         else:
-            self._data = SemiStrictDict.create(
-                allowed_keys=self._fields_ordered)()
+            self._data = {}
 
         self._dynamic_fields = SON()
 
diff --git a/tests/test_datastructures.py b/tests/test_datastructures.py
index 6830a188..79381c5a 100644
--- a/tests/test_datastructures.py
+++ b/tests/test_datastructures.py
@@ -1,6 +1,6 @@
 import unittest
 
-from mongoengine.base.datastructures import StrictDict, SemiStrictDict
+from mongoengine.base.datastructures import StrictDict
 
 
 class TestStrictDict(unittest.TestCase):
@@ -76,44 +76,5 @@ class TestStrictDict(unittest.TestCase):
         assert dict(**d) == {'a': 1, 'b': 2}
 
 
-class TestSemiSrictDict(TestStrictDict):
-    def strict_dict_class(self, *args, **kwargs):
-        return SemiStrictDict.create(*args, **kwargs)
-
-    def test_init_fails_on_nonexisting_attrs(self):
-        # disable irrelevant test
-        pass
-
-    def test_setattr_raises_on_nonexisting_attr(self):
-        # disable irrelevant test
-        pass
-
-    def test_setattr_getattr_nonexisting_attr_succeeds(self):
-        d = self.dtype()
-        d.x = 1
-        self.assertEqual(d.x, 1)
-
-    def test_init_succeeds_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        self.assertEqual((d.a, d.b, d.c, d.x), (1, 1, 1, 2))
-
-    def test_iter_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        self.assertEqual(list(d), ['a', 'b', 'c', 'x'])
-
-    def test_iteritems_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        self.assertEqual(list(d.iteritems()), [('a', 1), ('b', 1), ('c', 1), ('x', 2)])
-
-    def tets_cmp_with_strict_dicts(self):
-        d = self.dtype(a=1, b=1, c=1)
-        dd = StrictDict.create(("a", "b", "c"))(a=1, b=1, c=1)
-        self.assertEqual(d, dd)
-
-    def test_cmp_with_strict_dict_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        dd = StrictDict.create(("a", "b", "c", "x"))(a=1, b=1, c=1, x=2)
-        self.assertEqual(d, dd)
-
 if __name__ == '__main__':
     unittest.main()

From 66429ce331822c29e69770386785dfa0412dd02b Mon Sep 17 00:00:00 2001
From: Srinivas Reddy Thatiparthy <thatiparthysreenivas@gmail.com>
Date: Sun, 6 Aug 2017 13:35:01 +0530
Subject: [PATCH 017/121] Add python 3.6 support

---
 .travis.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 5e90660d..9362dcc2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,6 +15,7 @@ language: python
 python:
 - 2.7
 - 3.5
+- 3.6
 - pypy
 
 env:
@@ -39,6 +40,10 @@ matrix:
     env: MONGODB=2.4 PYMONGO=3.0
   - python: 3.5
     env: MONGODB=3.0 PYMONGO=3.0
+  - python: 3.6
+    env: MONGODB=2.4 PYMONGO=3.0
+  - python: 3.6
+    env: MONGODB=3.0 PYMONGO=3.0
 
 before_install:
 - bash .install_mongodb_on_travis.sh

From 7c0cfb1da2dd4bdcd24332a3ca08d561fa1e93da Mon Sep 17 00:00:00 2001
From: Srinivas Reddy Thatiparthy <thatiparthysreenivas@gmail.com>
Date: Sun, 6 Aug 2017 13:23:47 +0530
Subject: [PATCH 018/121] Add six.moves.range instead of xrange

---
 mongoengine/base/datastructures.py | 2 +-
 mongoengine/queryset/queryset.py   | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py
index b9aca8fa..c727ec78 100644
--- a/mongoengine/base/datastructures.py
+++ b/mongoengine/base/datastructures.py
@@ -127,7 +127,7 @@ class BaseList(list):
         return value
 
     def __iter__(self):
-        for i in xrange(self.__len__()):
+        for i in six.moves.range(self.__len__()):
             yield self[i]
 
     def __setitem__(self, key, value, *args, **kwargs):
diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py
index b5d2765b..cf913b01 100644
--- a/mongoengine/queryset/queryset.py
+++ b/mongoengine/queryset/queryset.py
@@ -1,3 +1,5 @@
+import six
+
 from mongoengine.errors import OperationError
 from mongoengine.queryset.base import (BaseQuerySet, CASCADE, DENY, DO_NOTHING,
                                        NULLIFY, PULL)
@@ -112,7 +114,7 @@ class QuerySet(BaseQuerySet):
         # Pull in ITER_CHUNK_SIZE docs from the database and store them in
         # the result cache.
         try:
-            for _ in xrange(ITER_CHUNK_SIZE):
+            for _ in six.moves.range(ITER_CHUNK_SIZE):
                 self._result_cache.append(self.next())
         except StopIteration:
             # Getting this exception means there are no more docs in the
@@ -166,7 +168,7 @@ class QuerySetNoCache(BaseQuerySet):
             return '.. queryset mid-iteration ..'
 
         data = []
-        for _ in xrange(REPR_OUTPUT_SIZE + 1):
+        for _ in six.moves.range(REPR_OUTPUT_SIZE + 1):
             try:
                 data.append(self.next())
             except StopIteration:

From 9b3fe09508e6bd5fc7fe8ed6fff97f515841bede Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Fri, 18 Aug 2017 09:31:26 +0800
Subject: [PATCH 019/121] added as_pymongo test for PointField #1311

---
 tests/queryset/geo.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py
index 51a32382..38c0377e 100644
--- a/tests/queryset/geo.py
+++ b/tests/queryset/geo.py
@@ -510,6 +510,24 @@ class GeoQueriesTest(MongoDBTestCase):
         roads = Road.objects.filter(poly__geo_intersects={"$geometry": polygon}).count()
         self.assertEqual(1, roads)
 
+    def test_aspymongo_with_only(self):
+        """Ensure as_pymongo works with only"""
+        class Place(Document):
+            location = PointField()
+
+        Place.drop_collection()
+        p = Place(location=[24.946861267089844, 60.16311983618494])
+        p.save()
+        qs = Place.objects().only('location')
+        self.assertDictEqual(
+            qs.as_pymongo()[0]['location'],
+            {u'type': u'Point',
+             u'coordinates': [
+                24.946861267089844,
+                60.16311983618494]
+            }
+        )
+
     def test_2dsphere_point_sets_correctly(self):
         class Location(Document):
             loc = PointField()

From dc8a64fa7d00364956560d47a7b753aee84fa561 Mon Sep 17 00:00:00 2001
From: Srinivas Reddy Thatiparthy <thatiparthysreenivas@gmail.com>
Date: Fri, 25 Aug 2017 22:02:47 +0530
Subject: [PATCH 020/121] Add missing dunder method  - __ne__ to the class
 GridFSProxy class

---
 mongoengine/fields.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 0d402712..fffba7ac 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -1465,6 +1465,9 @@ class GridFSProxy(object):
         else:
             return False
 
+    def __ne__(self, other):
+        return not self == other
+
     @property
     def fs(self):
         if not self._fs:

From 1eae97731f2a384cb7b27404aa9b59741155f7bc Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Wed, 30 Aug 2017 12:04:04 +0800
Subject: [PATCH 021/121] Fix Document.modify fail on sharded collection #1569

---
 mongoengine/document.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index f1622934..71929cf1 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -280,6 +280,9 @@ class Document(BaseDocument):
         elif query[id_field] != self.pk:
             raise InvalidQueryError('Invalid document modify query: it must modify only this document.')
 
+        # Need to add shard key to query, or you get an error
+        query.update(self._object_key)
+
         updated = self._qs(**query).modify(new=True, **update)
         if updated is None:
             return False

From 44732a5dd904da5224ae1ab7ab45ca4a247d82c1 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Sat, 2 Sep 2017 02:05:27 +0900
Subject: [PATCH 022/121] add fix for reload(fields) affect changed fields
 #1371

---
 mongoengine/document.py    |  5 +++--
 tests/document/instance.py | 20 +++++++++++++++++++-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index f1622934..cb901c82 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -702,7 +702,6 @@ class Document(BaseDocument):
             obj = obj[0]
         else:
             raise self.DoesNotExist('Document does not exist')
-
         for field in obj._data:
             if not fields or field in fields:
                 try:
@@ -718,7 +717,9 @@ class Document(BaseDocument):
                         # i.e. obj.update(unset__field=1) followed by obj.reload()
                         delattr(self, field)
 
-        self._changed_fields = obj._changed_fields
+        self._changed_fields = list(
+            set(self._changed_fields) - set(fields)
+            ) if fields else obj._changed_fields
         self._created = False
         return self
 
diff --git a/tests/document/instance.py b/tests/document/instance.py
index 609bc900..2a746709 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -476,6 +476,24 @@ class InstanceTest(unittest.TestCase):
         doc.save()
         doc.reload()
 
+    def test_reload_with_changed_fields(self):
+        """Ensures reloading will not affect changed fields"""
+        class User(Document):
+            name = StringField()
+            number = IntField()
+        User.drop_collection()
+
+        user = User(name="Bob", number=1).save()
+        user.name = "John"
+        user.number = 2
+
+        self.assertEqual(user._get_changed_fields(), ['name', 'number'])
+        user.reload('number')
+        self.assertEqual(user._get_changed_fields(), ['name'])
+        user.save()
+        user.reload()
+        self.assertEqual(user.name, "John")
+
     def test_reload_referencing(self):
         """Ensures reloading updates weakrefs correctly."""
         class Embedded(EmbeddedDocument):
@@ -521,7 +539,7 @@ class InstanceTest(unittest.TestCase):
         doc.save()
         doc.dict_field['extra'] = 1
         doc = doc.reload(10, 'list_field')
-        self.assertEqual(doc._get_changed_fields(), [])
+        self.assertEqual(doc._get_changed_fields(), ['dict_field.extra'])
         self.assertEqual(len(doc.list_field), 5)
         self.assertEqual(len(doc.dict_field), 3)
         self.assertEqual(len(doc.embedded_field.list_field), 4)

From 02733e6e5849bd60cd938fc05db2f73f8a561036 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Sat, 2 Sep 2017 12:00:57 +0900
Subject: [PATCH 023/121] fix flake8 error #1371

---
 mongoengine/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index cb901c82..a30dd0df 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -719,7 +719,7 @@ class Document(BaseDocument):
 
         self._changed_fields = list(
             set(self._changed_fields) - set(fields)
-            ) if fields else obj._changed_fields
+        ) if fields else obj._changed_fields
         self._created = False
         return self
 

From 70088704e202c38ed7265cbb1929a38d383c8a3d Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Sun, 10 Sep 2017 01:37:17 +0900
Subject: [PATCH 024/121] add tests to increase code coverage

---
 tests/fields/fields.py     | 10 +++++++++-
 tests/queryset/geo.py      |  4 ++++
 tests/queryset/queryset.py |  6 ++++++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/tests/fields/fields.py b/tests/fields/fields.py
index 7a0ccc25..dbf148e1 100644
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -334,7 +334,7 @@ class FieldTest(MongoDBTestCase):
     def test_string_validation(self):
         """Ensure that invalid values cannot be assigned to string fields."""
         class Person(Document):
-            name = StringField(max_length=20)
+            name = StringField(max_length=20, min_length=5)
             userid = StringField(r'[0-9a-z_]+$')
 
         person = Person(name=34)
@@ -352,6 +352,10 @@ class FieldTest(MongoDBTestCase):
         person = Person(name='Name that is more than twenty characters')
         self.assertRaises(ValidationError, person.validate)
 
+        # Test max length validation on name
+        person = Person(name='aa')
+        self.assertRaises(ValidationError, person.validate)
+
         person.name = 'Shorter name'
         person.validate()
 
@@ -437,6 +441,10 @@ class FieldTest(MongoDBTestCase):
         doc.age = 'ten'
         self.assertRaises(ValidationError, doc.validate)
 
+        # Test max_value validation
+        doc.value = 200
+        self.assertRaises(ValidationError, doc.validate)
+
     def test_float_validation(self):
         """Ensure that invalid values cannot be assigned to float fields.
         """
diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py
index 38c0377e..acfd9364 100644
--- a/tests/queryset/geo.py
+++ b/tests/queryset/geo.py
@@ -429,6 +429,10 @@ class GeoQueriesTest(MongoDBTestCase):
         roads = Road.objects.filter(line__geo_within=polygon).count()
         self.assertEqual(1, roads)
 
+        sphere = [[-1, 42,], 2]
+        roads = Road.objects.filter(line__geo_within_sphere=sphere).count()
+        self.assertEqual(0, roads)
+
         roads = Road.objects.filter(line__geo_within={"$geometry": polygon}).count()
         self.assertEqual(1, roads)
 
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index c78ed985..5fc73a54 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1882,6 +1882,12 @@ class QuerySetTest(unittest.TestCase):
         post.reload()
         self.assertTrue('mongo' in post.tags)
 
+        # Push with arrays
+        BlogPost.objects.update(push__tags=['python', 'scala'])
+        post.reload()
+        self.assertTrue('python' in post.tags)
+        self.assertTrue('scala' in post.tags)
+
         BlogPost.objects.update_one(push_all__tags=['db', 'nosql'])
         post.reload()
         self.assertTrue('db' in post.tags and 'nosql' in post.tags)

From ba99190f535ba16ed3c9d6fe5acfc4836c64b48d Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Sun, 10 Sep 2017 13:09:20 +0900
Subject: [PATCH 025/121] add tests to increase coverage

---
 tests/fields/fields.py     | 26 ++++++++++++++++++++++++++
 tests/queryset/queryset.py |  4 ++++
 2 files changed, 30 insertions(+)

diff --git a/tests/fields/fields.py b/tests/fields/fields.py
index dbf148e1..d25f2e25 100644
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -312,6 +312,27 @@ class FieldTest(MongoDBTestCase):
 
         self.assertEqual(1, TestDocument.objects(long_fld__ne=None).count())
 
+    def test_callable_validation(self):
+        """Ensure that callable validation works"""
+        def check_even(value):
+            return value % 2 == 0
+
+        class Order(Document):
+            number = IntField(validation=check_even)
+
+        Order.drop_collection()
+
+        order = Order(number=3)
+        self.assertRaises(ValidationError, order.validate)
+
+        class User(Document):
+            name = StringField(validation=1)
+
+        User.drop_collection()
+
+        user = User(name='test')
+        self.assertRaises(ValidationError, user.validate)
+
     def test_object_id_validation(self):
         """Ensure that invalid values cannot be assigned to an
         ObjectIdField.
@@ -527,6 +548,11 @@ class FieldTest(MongoDBTestCase):
             class User(Document):
                 name = StringField(db_field='name\0')
 
+        # db field should be a string
+        with self.assertRaises(TypeError):
+            class User(Document):
+                name = StringField(db_field=1)
+
     def test_decimal_comparison(self):
         class Person(Document):
             money = DecimalField()
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 5fc73a54..9c61513a 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1861,6 +1861,10 @@ class QuerySetTest(unittest.TestCase):
         post = BlogPost(name="Test Post", hits=5, tags=['test'])
         post.save()
 
+        BlogPost.objects.update(hits=11)
+        post.reload()
+        self.assertEqual(post.hits, 11)
+
         BlogPost.objects.update(set__hits=10)
         post.reload()
         self.assertEqual(post.hits, 10)

From be8f1b9fdd522888527f871c1fbc2a0f6f0af279 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Thu, 14 Sep 2017 22:24:27 +0900
Subject: [PATCH 026/121] add failing test for generic_emdedded_document query
 #1651

---
 tests/queryset/queryset.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index c78ed985..4bc5fef6 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -4790,6 +4790,30 @@ class QuerySetTest(unittest.TestCase):
         for obj in C.objects.no_sub_classes():
             self.assertEqual(obj.__class__, C)
 
+    def test_query_generic_embedded_document(self):
+        """Ensure that querying sub field on generic_embedded_field works
+        """
+        class A(EmbeddedDocument):
+            a_name = StringField()
+
+        class B(EmbeddedDocument):
+            b_name = StringField()
+
+        class Doc(Document):
+            document = GenericEmbeddedDocumentField(choices=(A, B))
+
+        Doc.drop_collection()
+        Doc(document=A(a_name='A doc')).save()
+        Doc(document=B(b_name='B doc')).save()
+
+        # Using raw in filter working fine
+        self.assertEqual(Doc.objects(
+            __raw__={'document.a_name': 'A doc'}).count(), 1)
+        self.assertEqual(Doc.objects(
+            __raw__={'document.b_name': 'B doc'}).count(), 1)
+        self.assertEqual(Doc.objects(document__a_name='A doc').count(), 1)
+        self.assertEqual(Doc.objects(document__b_name='B doc').count(), 1)
+
     def test_query_reference_to_custom_pk_doc(self):
 
         class A(Document):

From e90f6a2fa3766872b44edad29a21527659984ae3 Mon Sep 17 00:00:00 2001
From: Andy Yankovsky <weratt@gmail.com>
Date: Thu, 14 Sep 2017 20:28:15 +0300
Subject: [PATCH 027/121] Fix update via pull__something__in=[]

---
 mongoengine/queryset/transform.py | 46 +++++++++++++++++++------------
 tests/queryset/transform.py       |  7 +++++
 2 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index a9907ada..1f70a48b 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -101,21 +101,8 @@ def query(_doc_cls=None, **kwargs):
                         value = value['_id']
 
             elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict):
-                # Raise an error if the in/nin/all/near param is not iterable. We need a
-                # special check for BaseDocument, because - although it's iterable - using
-                # it as such in the context of this method is most definitely a mistake.
-                BaseDocument = _import_class('BaseDocument')
-                if isinstance(value, BaseDocument):
-                    raise TypeError("When using the `in`, `nin`, `all`, or "
-                                    "`near`-operators you can\'t use a "
-                                    "`Document`, you must wrap your object "
-                                    "in a list (object -> [object]).")
-                elif not hasattr(value, '__iter__'):
-                    raise TypeError("The `in`, `nin`, `all`, or "
-                                    "`near`-operators must be applied to an "
-                                    "iterable (e.g. a list).")
-                else:
-                    value = [field.prepare_query_value(op, v) for v in value]
+                # Raise an error if the in/nin/all/near param is not iterable.
+                value = _prepare_query_for_iterable(field, op, value)
 
             # If we're querying a GenericReferenceField, we need to alter the
             # key depending on the value:
@@ -284,9 +271,15 @@ def update(_doc_cls=None, **update):
             if isinstance(field, GeoJsonBaseField):
                 value = field.to_mongo(value)
 
-            if op == 'push' and isinstance(value, (list, tuple, set)):
+            if op == 'pull':
+                if field.required or value is not None:
+                    if match == 'in' and not isinstance(value, dict):
+                        value = _prepare_query_for_iterable(field, op, value)
+                    else:
+                        value = field.prepare_query_value(op, value)
+            elif op == 'push' and isinstance(value, (list, tuple, set)):
                 value = [field.prepare_query_value(op, v) for v in value]
-            elif op in (None, 'set', 'push', 'pull'):
+            elif op in (None, 'set', 'push'):
                 if field.required or value is not None:
                     value = field.prepare_query_value(op, value)
             elif op in ('pushAll', 'pullAll'):
@@ -439,3 +432,22 @@ def _infer_geometry(value):
 
     raise InvalidQueryError('Invalid $geometry data. Can be either a '
                             'dictionary or (nested) lists of coordinate(s)')
+
+
+def _prepare_query_for_iterable(field, op, value):
+    # We need a special check for BaseDocument, because - although it's iterable - using
+    # it as such in the context of this method is most definitely a mistake.
+    BaseDocument = _import_class('BaseDocument')
+
+    if isinstance(value, BaseDocument):
+        raise TypeError("When using the `in`, `nin`, `all`, or "
+                        "`near`-operators you can\'t use a "
+                        "`Document`, you must wrap your object "
+                        "in a list (object -> [object]).")
+
+    if not hasattr(value, '__iter__'):
+        raise TypeError("The `in`, `nin`, `all`, or "
+                        "`near`-operators must be applied to an "
+                        "iterable (e.g. a list).")
+
+    return [field.prepare_query_value(op, v) for v in value]
diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py
index 20ab0b3f..a043a647 100644
--- a/tests/queryset/transform.py
+++ b/tests/queryset/transform.py
@@ -28,12 +28,16 @@ class TransformTest(unittest.TestCase):
                          {'name': {'$exists': True}})
 
     def test_transform_update(self):
+        class LisDoc(Document):
+            foo = ListField(StringField())
+
         class DicDoc(Document):
             dictField = DictField()
 
         class Doc(Document):
             pass
 
+        LisDoc.drop_collection()
         DicDoc.drop_collection()
         Doc.drop_collection()
 
@@ -51,6 +55,9 @@ class TransformTest(unittest.TestCase):
         update = transform.update(DicDoc, pull__dictField__test=doc)
         self.assertTrue(isinstance(update["$pull"]["dictField"]["test"], dict))
 
+        update = transform.update(LisDoc, pull__foo__in=['a'])
+        self.assertEqual(update, {'$pull': {'foo': {'$in': ['a']}}})
+
     def test_query_field_name(self):
         """Ensure that the correct field name is used when querying.
         """

From 2f4e2bde6bb1d892ac22928870e01889fa983833 Mon Sep 17 00:00:00 2001
From: Andy Yankovsky <weratt@gmail.com>
Date: Thu, 14 Sep 2017 21:02:53 +0300
Subject: [PATCH 028/121] Update AUTHORS

---
 AUTHORS | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index 4eac5eb2..2e7b56fc 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -244,4 +244,5 @@ that much better:
  * Stanislav Kaledin (https://github.com/sallyruthstruik)
  * Dmitry Yantsen (https://github.com/mrTable)
  * Renjianxin (https://github.com/Davidrjx)
- * Erdenezul Batmunkh (https://github.com/erdenezul)
\ No newline at end of file
+ * Erdenezul Batmunkh (https://github.com/erdenezul)
+ * Andy Yankovsky (https://github.com/werat)

From aa4996ef28bec2614cb11c055b5d184fee6b9549 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Fri, 15 Sep 2017 11:18:24 +0800
Subject: [PATCH 029/121] fix bug query subfield in generic_embedded_document
 #1651

---
 mongoengine/fields.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index fffba7ac..cc66008b 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -692,6 +692,14 @@ class GenericEmbeddedDocumentField(BaseField):
 
         value.validate(clean=clean)
 
+    def lookup_member(self, member_name):
+        if self.choices:
+            for choice in self.choices:
+                field = choice._fields.get(member_name)
+                if field:
+                    return field
+        return None
+
     def to_mongo(self, document, use_db_field=True, fields=None):
         if document is None:
             return None

From 091a02f73737ccd028aa256e2f50cc30d84621a7 Mon Sep 17 00:00:00 2001
From: Stefan Wojcik <wojcikstefan@gmail.com>
Date: Sun, 1 Oct 2017 16:09:10 -0400
Subject: [PATCH 030/121] minor .travis.yml comment correction [ci skip]

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 9362dcc2..78a9f787 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -97,7 +97,7 @@ deploy:
   distributions: "sdist bdist_wheel"
 
   # only deploy on tagged commits (aka GitHub releases) and only for the
-  # parent repo's builds running Python 2.7 along with dev PyMongo (we run
+  # parent repo's builds running Python 2.7 along with PyMongo v3.0 (we run
   # Travis against many different Python and PyMongo versions and we don't
   # want the deploy to occur multiple times).
   on:

From 01526a7b373b954ef053238331f84ef1193bb564 Mon Sep 17 00:00:00 2001
From: Stefan Wojcik <wojcikstefan@gmail.com>
Date: Sun, 1 Oct 2017 16:32:02 -0400
Subject: [PATCH 031/121] v0.14.1 version bump + updated changelog

---
 docs/changelog.rst      | 10 +++++++++-
 mongoengine/__init__.py |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 1dcbfb51..f04ab314 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,7 +4,15 @@ Changelog
 
 Development
 ===========
-- Improve performances by removing SemiStrictDict
+- (Fill this out as you fix issues and develop your features).
+
+Changes in 0.14.1
+=================
+- Removed SemiStrictDict and started using a regular dict for `BaseDocument._data` #1630
+- Added support for the `$position` param in the `$push` operator #1566
+- Fixed `DateTimeField` interpreting an empty string as today #1533
+- Added a missing `__ne__` method to the `GridFSProxy` class #1632
+- Fixed `BaseQuerySet._fields_to_db_fields` #1553
 
 Changes in 0.14.0
 =================
diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py
index b41e87e7..4b60f1b7 100644
--- a/mongoengine/__init__.py
+++ b/mongoengine/__init__.py
@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
            list(signals.__all__) + list(errors.__all__))
 
 
-VERSION = (0, 14, 0)
+VERSION = (0, 14, 1)
 
 
 def get_version():

From d79ab5ffeb421abe4c145af77633283f4d20d942 Mon Sep 17 00:00:00 2001
From: Stefan Wojcik <wojcikstefan@gmail.com>
Date: Sun, 1 Oct 2017 17:05:28 -0400
Subject: [PATCH 032/121] remove {nospam} from author_email & maintainer_email
 (PyPI doesnt validate those anymore)

---
 setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index fa682d20..98964d19 100644
--- a/setup.py
+++ b/setup.py
@@ -70,9 +70,9 @@ setup(
     name='mongoengine',
     version=VERSION,
     author='Harry Marr',
-    author_email='harry.marr@{nospam}gmail.com',
-    maintainer="Ross Lawley",
-    maintainer_email="ross.lawley@{nospam}gmail.com",
+    author_email='harry.marr@gmail.com',
+    maintainer="Stefan Wojcik",
+    maintainer_email="wojcikstefan@gmail.com",
     url='http://mongoengine.org/',
     download_url='https://github.com/MongoEngine/mongoengine/tarball/master',
     license='MIT',

From a1494c4c93cce608d8d9450c9ef76c07559248c9 Mon Sep 17 00:00:00 2001
From: Stefan Wojcik <wojcikstefan@gmail.com>
Date: Sun, 1 Oct 2017 17:31:10 -0400
Subject: [PATCH 033/121] v0.14.3 version bump

---
 mongoengine/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py
index 4b60f1b7..c7c6f707 100644
--- a/mongoengine/__init__.py
+++ b/mongoengine/__init__.py
@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
            list(signals.__all__) + list(errors.__all__))
 
 
-VERSION = (0, 14, 1)
+VERSION = (0, 14, 3)
 
 
 def get_version():

From 9ab856e186b438bc28f12cae8274ea1bb2ff3614 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Tue, 10 Oct 2017 10:34:34 +0800
Subject: [PATCH 034/121] use each modifier only with  #1673

---
 mongoengine/queryset/transform.py |  2 --
 tests/document/instance.py        | 11 +++++++++++
 tests/queryset/queryset.py        | 15 +++++++++++++++
 3 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index a9907ada..3eadaf64 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -344,8 +344,6 @@ def update(_doc_cls=None, **update):
                 if not isinstance(value, (set, tuple, list)):
                     value = [value]
                 value = {key: {'$each': value, '$position': position}}
-            elif isinstance(value, list):
-                value = {key: {'$each': value}}
             else:
                 value = {key: value}
         else:
diff --git a/tests/document/instance.py b/tests/document/instance.py
index 609bc900..721aa95f 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -3183,6 +3183,17 @@ class InstanceTest(unittest.TestCase):
         blog.reload()
         self.assertEqual(blog.tags, ['mongodb', 'code', 'python'])
 
+    def test_push_nested_list(self):
+        """Ensure that push update works in nested list"""
+        class BlogPost(Document):
+            slug = StringField()
+            tags = ListField()
+
+        blog = BlogPost(slug="test").save()
+        blog.update(push__tags=["value1", 123])
+        blog.reload()
+        self.assertEqual(blog.tags, [["value1", 123]])
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index c78ed985..d633b8b2 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1929,6 +1929,21 @@ class QuerySetTest(unittest.TestCase):
         post.reload()
         self.assertEqual(post.tags, ['scala', 'mongodb', 'python', 'java'])
 
+    def test_update_push_list_of_list(self):
+        """Ensure that the 'push' update operation works in the list of list
+        """
+        class BlogPost(Document):
+            slug = StringField()
+            tags = ListField()
+
+        BlogPost.drop_collection()
+
+        post = BlogPost(slug="test").save()
+
+        BlogPost.objects.filter(slug="test").update(push__tags=["value1", 123])
+        post.reload()
+        self.assertEqual(post.tags, [["value1", 123]])
+
     def test_update_push_and_pull_add_to_set(self):
         """Ensure that the 'pull' update operation works correctly.
         """

From e6c0280b40f18074c0a6267b29f2a77f21c1c779 Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Mon, 30 Oct 2017 18:15:51 +0100
Subject: [PATCH 035/121] Add LazyReferenceField

---
 docs/changelog.rst                 |   6 +-
 mongoengine/base/__init__.py       |   2 +-
 mongoengine/base/datastructures.py |  42 ++++-
 mongoengine/fields.py              | 139 ++++++++++++++-
 tests/fields/fields.py             | 276 ++++++++++++++++++++++++++++-
 5 files changed, 456 insertions(+), 9 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index f04ab314..834fbee2 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -2,9 +2,9 @@
 Changelog
 =========
 
-Development
-===========
-- (Fill this out as you fix issues and develop your features).
+Changes in 0.15.0
+=================
+- Add LazyReferenceField to address #1230
 
 Changes in 0.14.1
 =================
diff --git a/mongoengine/base/__init__.py b/mongoengine/base/__init__.py
index da31b922..e069a147 100644
--- a/mongoengine/base/__init__.py
+++ b/mongoengine/base/__init__.py
@@ -15,7 +15,7 @@ __all__ = (
     'UPDATE_OPERATORS', '_document_registry', 'get_document',
 
     # datastructures
-    'BaseDict', 'BaseList', 'EmbeddedDocumentList',
+    'BaseDict', 'BaseList', 'EmbeddedDocumentList', 'LazyReference',
 
     # document
     'BaseDocument',
diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py
index 14fe95e9..043df471 100644
--- a/mongoengine/base/datastructures.py
+++ b/mongoengine/base/datastructures.py
@@ -2,11 +2,12 @@ import itertools
 import weakref
 
 import six
+from bson import DBRef
 
 from mongoengine.common import _import_class
 from mongoengine.errors import DoesNotExist, MultipleObjectsReturned
 
-__all__ = ('BaseDict', 'BaseList', 'EmbeddedDocumentList')
+__all__ = ('BaseDict', 'BaseList', 'EmbeddedDocumentList', 'LazyReference')
 
 
 class BaseDict(dict):
@@ -445,3 +446,42 @@ class StrictDict(object):
 
             cls._classes[allowed_keys] = SpecificStrictDict
         return cls._classes[allowed_keys]
+
+
+class LazyReference(DBRef):
+    __slots__ = ('_cached_doc', 'passthrough', 'document_type')
+
+    def fetch(self, force=False):
+        if not self._cached_doc or force:
+            self._cached_doc = self.document_type.objects.get(pk=self.pk)
+            if not self._cached_doc:
+                raise DoesNotExist('Trying to dereference unknown document %s' % (self))
+        return self._cached_doc
+
+    @property
+    def pk(self):
+        return self.id
+
+    def __init__(self, document_type, pk, cached_doc=None, passthrough=False):
+        self.document_type = document_type
+        self._cached_doc = cached_doc
+        self.passthrough = passthrough
+        super(LazyReference, self).__init__(self.document_type._get_collection_name(), pk)
+
+    def __getitem__(self, name):
+        if not self.passthrough:
+            raise KeyError()
+        document = self.fetch()
+        return document[name]
+
+    def __getattr__(self, name):
+        if not object.__getattribute__(self, 'passthrough'):
+            raise AttributeError()
+        document = self.fetch()
+        try:
+            return document[name]
+        except KeyError:
+            raise AttributeError()
+
+    def __repr__(self):
+        return "<LazyReference(%s, %r)>" % (self.document_type, self.pk)
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index fffba7ac..73a62bc5 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -26,7 +26,8 @@ except ImportError:
     Int64 = long
 
 from mongoengine.base import (BaseDocument, BaseField, ComplexBaseField,
-                              GeoJsonBaseField, ObjectIdField, get_document)
+                              GeoJsonBaseField, ObjectIdField, get_document,
+                              LazyReference)
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
@@ -46,6 +47,8 @@ __all__ = (
     'GenericEmbeddedDocumentField', 'DynamicField', 'ListField',
     'SortedListField', 'EmbeddedDocumentListField', 'DictField',
     'MapField', 'ReferenceField', 'CachedReferenceField',
+    'LazyReferenceField',
+    # 'GenericLazyReferenceField',
     'GenericReferenceField', 'BinaryField', 'GridFSError', 'GridFSProxy',
     'FileField', 'ImageGridFsProxy', 'ImproperlyConfigured', 'ImageField',
     'GeoPointField', 'PointField', 'LineStringField', 'PolygonField',
@@ -953,6 +956,15 @@ class ReferenceField(BaseField):
     """A reference to a document that will be automatically dereferenced on
     access (lazily).
 
+    Note this means you will get a database I/O access everytime you access
+    this field. This is necessary because the field returns a :class:`~mongoengine.Document`
+    which precise type can depend of the value of the `_cls` field present in the
+    document in database.
+    In short, using this type of field can lead to poor performances (especially
+    if you access this field only to retrieve it `pk` field which is already
+    known before dereference). To solve this you should consider using the
+    :class:`~mongoengine.fields.LazyReferenceField`.
+
     Use the `reverse_delete_rule` to handle what should happen if the document
     the field is referencing is deleted.  EmbeddedDocuments, DictFields and
     MapFields does not support reverse_delete_rule and an `InvalidDocumentError`
@@ -1087,8 +1099,8 @@ class ReferenceField(BaseField):
 
     def validate(self, value):
 
-        if not isinstance(value, (self.document_type, DBRef, ObjectId)):
-            self.error('A ReferenceField only accepts DBRef, ObjectId or documents')
+        if not isinstance(value, (self.document_type, LazyReference, DBRef, ObjectId)):
+            self.error('A ReferenceField only accepts DBRef, LazyReference, ObjectId or documents')
 
         if isinstance(value, Document) and value.id is None:
             self.error('You can only reference documents once they have been '
@@ -2141,3 +2153,124 @@ class MultiPolygonField(GeoJsonBaseField):
     .. versionadded:: 0.9
     """
     _type = 'MultiPolygon'
+
+
+class LazyReferenceField(BaseField):
+    """A really lazy reference to a document.
+    Unlike the :class:`~mongoengine.fields.ReferenceField` it must be manually
+    dereferenced using it ``fetch()`` method.
+    """
+
+    def __init__(self, document_type, passthrough=False, dbref=False,
+                 reverse_delete_rule=DO_NOTHING, **kwargs):
+        """Initialises the Reference Field.
+
+        :param dbref:  Store the reference as :class:`~pymongo.dbref.DBRef`
+          or as the :class:`~pymongo.objectid.ObjectId`.id .
+        :param reverse_delete_rule: Determines what to do when the referring
+          object is deleted
+        :param passthrough: When trying to access unknown fields, the
+        :class:`~mongoengine.base.datastructure.LazyReference` instance will
+        automatically call `fetch()` and try to retrive the field on the fetched
+        document. Note this only work getting field (not setting or deleting).
+        """
+        if (
+            not isinstance(document_type, six.string_types) and
+            not issubclass(document_type, Document)
+        ):
+            self.error('Argument to LazyReferenceField constructor must be a '
+                       'document class or a string')
+
+        self.dbref = dbref
+        self.passthrough = passthrough
+        self.document_type_obj = document_type
+        self.reverse_delete_rule = reverse_delete_rule
+        super(LazyReferenceField, self).__init__(**kwargs)
+
+    @property
+    def document_type(self):
+        if isinstance(self.document_type_obj, six.string_types):
+            if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT:
+                self.document_type_obj = self.owner_document
+            else:
+                self.document_type_obj = get_document(self.document_type_obj)
+        return self.document_type_obj
+
+    def __get__(self, instance, owner):
+        """Descriptor to allow lazy dereferencing."""
+        if instance is None:
+            # Document class being used rather than a document object
+            return self
+
+        value = instance._data.get(self.name)
+        if isinstance(value, LazyReference):
+            if value.passthrough != self.passthrough:
+                instance._data[self.name] = LazyReference(
+                    value.document_type, value.pk, passthrough=self.passthrough)
+        elif value is not None:
+            if isinstance(value, self.document_type):
+                value = LazyReference(self.document_type, value.pk, passthrough=self.passthrough)
+            elif isinstance(value, DBRef):
+                value = LazyReference(self.document_type, value.id, passthrough=self.passthrough)
+            else:
+                # value is the primary key of the referenced document
+                value = LazyReference(self.document_type, value, passthrough=self.passthrough)
+            instance._data[self.name] = value
+
+        return super(LazyReferenceField, self).__get__(instance, owner)
+
+    def to_mongo(self, value):
+        if isinstance(value, LazyReference):
+            pk = value.pk
+        elif isinstance(value, self.document_type):
+            pk = value.pk
+        elif isinstance(value, DBRef):
+            pk = value.id
+        else:
+            # value is the primary key of the referenced document
+            pk = value
+        id_field_name = self.document_type._meta['id_field']
+        id_field = self.document_type._fields[id_field_name]
+        pk = id_field.to_mongo(pk)
+        if self.dbref:
+            return DBRef(self.document_type._get_collection_name(), pk)
+        else:
+            return pk
+
+    def validate(self, value):
+        if isinstance(value, LazyReference):
+            if not issubclass(value.document_type, self.document_type):
+                self.error('Reference must be on a `%s` document.' % self.document_type)
+            pk = value.pk
+        elif isinstance(value, self.document_type):
+            pk = value.pk
+        elif isinstance(value, DBRef):
+            # TODO: check collection ?
+            collection = self.document_type._get_collection_name()
+            if value.collection != collection:
+                self.error("DBRef on bad collection (must be on `%s`)" % collection)
+            pk = value.id
+        else:
+            # value is the primary key of the referenced document
+            id_field_name = self.document_type._meta['id_field']
+            id_field = getattr(self.document_type, id_field_name)
+            pk = value
+            try:
+                id_field.validate(pk)
+            except ValidationError:
+                self.error("value should be `{0}` document, LazyReference or DBRef on `{0}` "
+                           "or `{0}`'s primary key (i.e. `{1}`)".format(
+                            self.document_type.__name__, type(id_field).__name__))
+
+        if pk is None:
+            self.error('You can only reference documents once they have been '
+                       'saved to the database')
+
+    def prepare_query_value(self, op, value):
+        if value is None:
+            return None
+        super(LazyReferenceField, self).prepare_query_value(op, value)
+        return self.to_mongo(value)
+
+    def lookup_member(self, member_name):
+        return self.document_type._fields.get(member_name)
diff --git a/tests/fields/fields.py b/tests/fields/fields.py
index 7a0ccc25..84156622 100644
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -26,7 +26,7 @@ except ImportError:
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.base import (BaseDict, BaseField, EmbeddedDocumentList,
-                              _document_registry)
+                              _document_registry, LazyReference)
 
 from tests.utils import MongoDBTestCase
 
@@ -931,7 +931,9 @@ class FieldTest(MongoDBTestCase):
             comments = ListField(EmbeddedDocumentField(Comment))
             tags = ListField(StringField())
             authors = ListField(ReferenceField(User))
+            authors_as_lazy = ListField(LazyReferenceField(User))
             generic = ListField(GenericReferenceField())
+            # generic_as_lazy = ListField(LazyGenericReferenceField())
 
         User.drop_collection()
         BlogPost.drop_collection()
@@ -969,6 +971,15 @@ class FieldTest(MongoDBTestCase):
         post.authors = [user]
         post.validate()
 
+        post.authors_as_lazy = [Comment()]
+        self.assertRaises(ValidationError, post.validate)
+
+        post.authors_as_lazy = [User()]
+        self.assertRaises(ValidationError, post.validate)
+
+        post.authors_as_lazy = [user]
+        post.validate()
+
         post.generic = [1, 2]
         self.assertRaises(ValidationError, post.validate)
 
@@ -981,6 +992,18 @@ class FieldTest(MongoDBTestCase):
         post.generic = [user]
         post.validate()
 
+        # post.generic_as_lazy = [1, 2]
+        # self.assertRaises(ValidationError, post.validate)
+
+        # post.generic_as_lazy = [User(), Comment()]
+        # self.assertRaises(ValidationError, post.validate)
+
+        # post.generic_as_lazy = [Comment()]
+        # self.assertRaises(ValidationError, post.validate)
+
+        # post.generic_as_lazy = [user]
+        # post.validate()
+
     def test_sorted_list_sorting(self):
         """Ensure that a sorted list field properly sorts values.
         """
@@ -4598,5 +4621,256 @@ class CachedReferenceFieldTest(MongoDBTestCase):
         self.assertTrue(isinstance(ocorrence.animal, Animal))
 
 
+class LazyReferenceFieldTest(MongoDBTestCase):
+    def test_lazy_reference_config(self):
+        # Make sure ReferenceField only accepts a document class or a string
+        # with a document class name.
+        self.assertRaises(ValidationError, LazyReferenceField, EmbeddedDocument)
+
+    def test_lazy_reference_simple(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            person = StringField()
+            animal = LazyReferenceField(Animal)
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal = Animal(name="Leopard", tag="heavy").save()
+        Ocurrence(person="test", animal=animal).save()
+        p = Ocurrence.objects.get()
+        self.assertIsInstance(p.animal, LazyReference)
+        fetched_animal = p.animal.fetch()
+        self.assertEqual(fetched_animal, animal)
+        # `fetch` keep cache on referenced document by default...
+        animal.tag = "not so heavy"
+        animal.save()
+        double_fetch = p.animal.fetch()
+        self.assertIs(fetched_animal, double_fetch)
+        self.assertEqual(double_fetch.tag, "heavy")
+        # ...unless specified otherwise
+        fetch_force = p.animal.fetch(force=True)
+        self.assertIsNot(fetch_force, fetched_animal)
+        self.assertEqual(fetch_force.tag, "not so heavy")
+
+    def test_lazy_reference_fetch_invalid_ref(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            person = StringField()
+            animal = LazyReferenceField(Animal)
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal = Animal(name="Leopard", tag="heavy").save()
+        Ocurrence(person="test", animal=animal).save()
+        animal.delete()
+        p = Ocurrence.objects.get()
+        self.assertIsInstance(p.animal, LazyReference)
+        with self.assertRaises(DoesNotExist):
+            p.animal.fetch()
+
+    def test_lazy_reference_set(self):
+        class Animal(Document):
+            meta = {'allow_inheritance': True}
+
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            person = StringField()
+            animal = LazyReferenceField(Animal)
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        class SubAnimal(Animal):
+            nick = StringField()
+
+        animal = Animal(name="Leopard", tag="heavy").save()
+        sub_animal = SubAnimal(nick='doggo', name='dog').save()
+        for ref in (
+                animal,
+                animal.pk,
+                DBRef(animal._get_collection_name(), animal.pk),
+                LazyReference(Animal, animal.pk),
+
+                sub_animal,
+                sub_animal.pk,
+                DBRef(sub_animal._get_collection_name(), sub_animal.pk),
+                LazyReference(SubAnimal, sub_animal.pk),
+                ):
+            p = Ocurrence(person="test", animal=ref).save()
+            p.reload()
+            self.assertIsInstance(p.animal, LazyReference)
+            p.animal.fetch()
+
+    def test_lazy_reference_bad_set(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            person = StringField()
+            animal = LazyReferenceField(Animal)
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        class BadDoc(Document):
+            pass
+
+        animal = Animal(name="Leopard", tag="heavy").save()
+        baddoc = BadDoc().save()
+        for bad in (
+                42,
+                'foo',
+                baddoc,
+                DBRef(baddoc._get_collection_name(), animal.pk),
+                LazyReference(BadDoc, animal.pk)
+                ):
+            with self.assertRaises(ValidationError):
+                p = Ocurrence(person="test", animal=bad).save()
+
+    def test_lazy_reference_query_conversion(self):
+        """Ensure that LazyReferenceFields can be queried using objects and values
+        of the type of the primary key of the referenced object.
+        """
+        class Member(Document):
+            user_num = IntField(primary_key=True)
+
+        class BlogPost(Document):
+            title = StringField()
+            author = LazyReferenceField(Member, dbref=False)
+
+        Member.drop_collection()
+        BlogPost.drop_collection()
+
+        m1 = Member(user_num=1)
+        m1.save()
+        m2 = Member(user_num=2)
+        m2.save()
+
+        post1 = BlogPost(title='post 1', author=m1)
+        post1.save()
+
+        post2 = BlogPost(title='post 2', author=m2)
+        post2.save()
+
+        post = BlogPost.objects(author=m1).first()
+        self.assertEqual(post.id, post1.id)
+
+        post = BlogPost.objects(author=m2).first()
+        self.assertEqual(post.id, post2.id)
+
+        # Same thing by passing a LazyReference instance
+        post = BlogPost.objects(author=LazyReference(Member, m2.pk)).first()
+        self.assertEqual(post.id, post2.id)
+
+    def test_lazy_reference_query_conversion_dbref(self):
+        """Ensure that LazyReferenceFields can be queried using objects and values
+        of the type of the primary key of the referenced object.
+        """
+        class Member(Document):
+            user_num = IntField(primary_key=True)
+
+        class BlogPost(Document):
+            title = StringField()
+            author = LazyReferenceField(Member, dbref=True)
+
+        Member.drop_collection()
+        BlogPost.drop_collection()
+
+        m1 = Member(user_num=1)
+        m1.save()
+        m2 = Member(user_num=2)
+        m2.save()
+
+        post1 = BlogPost(title='post 1', author=m1)
+        post1.save()
+
+        post2 = BlogPost(title='post 2', author=m2)
+        post2.save()
+
+        post = BlogPost.objects(author=m1).first()
+        self.assertEqual(post.id, post1.id)
+
+        post = BlogPost.objects(author=m2).first()
+        self.assertEqual(post.id, post2.id)
+
+        # Same thing by passing a LazyReference instance
+        post = BlogPost.objects(author=LazyReference(Member, m2.pk)).first()
+        self.assertEqual(post.id, post2.id)
+
+    def test_lazy_reference_passthrough(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            animal = LazyReferenceField(Animal, passthrough=False)
+            animal_passthrough = LazyReferenceField(Animal, passthrough=True)
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal = Animal(name="Leopard", tag="heavy").save()
+        Ocurrence(animal=animal, animal_passthrough=animal).save()
+        p = Ocurrence.objects.get()
+        self.assertIsInstance(p.animal, LazyReference)
+        with self.assertRaises(KeyError):
+            p.animal['name']
+        with self.assertRaises(AttributeError):
+            p.animal.name
+        self.assertEqual(p.animal.pk, animal.pk)
+
+        self.assertEqual(p.animal_passthrough.name, "Leopard")
+        self.assertEqual(p.animal_passthrough['name'], "Leopard")
+
+        # Should not be able to access referenced document's methods
+        with self.assertRaises(AttributeError):
+            p.animal.save
+        with self.assertRaises(KeyError):
+            p.animal['save']
+
+    def test_lazy_reference_not_set(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            person = StringField()
+            animal = LazyReferenceField(Animal)
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        Ocurrence(person='foo').save()
+        p = Ocurrence.objects.get()
+        self.assertIs(p.animal, None)
+
+    def test_lazy_reference_equality(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        Animal.drop_collection()
+
+        animal = Animal(name="Leopard", tag="heavy").save()
+        animalref = LazyReference(Animal, animal.pk)
+        self.assertEqual(animal, animalref)
+        self.assertEqual(animalref, animal)
+
+        other_animalref = LazyReference(Animal, ObjectId("54495ad94c934721ede76f90"))
+        self.assertNotEqual(animal, other_animalref)
+        self.assertNotEqual(other_animalref, animal)
+
+
 if __name__ == '__main__':
     unittest.main()

From 35d04582280c189ce7433864686fda20ae859b02 Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Mon, 6 Nov 2017 12:17:31 +0100
Subject: [PATCH 036/121] Add GenericLazyReferenceField

---
 docs/changelog.rst     |   2 +-
 mongoengine/fields.py  |  73 ++++++++++++++-
 tests/fields/fields.py | 198 +++++++++++++++++++++++++++++++++++++++--
 3 files changed, 261 insertions(+), 12 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 834fbee2..1e9ac7fc 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,7 +4,7 @@ Changelog
 
 Changes in 0.15.0
 =================
-- Add LazyReferenceField to address #1230
+- Add LazyReferenceField and GenericLazyReferenceField to address #1230
 
 Changes in 0.14.1
 =================
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 73a62bc5..61e0cb69 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -47,8 +47,7 @@ __all__ = (
     'GenericEmbeddedDocumentField', 'DynamicField', 'ListField',
     'SortedListField', 'EmbeddedDocumentListField', 'DictField',
     'MapField', 'ReferenceField', 'CachedReferenceField',
-    'LazyReferenceField',
-    # 'GenericLazyReferenceField',
+    'LazyReferenceField', 'GenericLazyReferenceField',
     'GenericReferenceField', 'BinaryField', 'GridFSError', 'GridFSProxy',
     'FileField', 'ImageGridFsProxy', 'ImproperlyConfigured', 'ImageField',
     'GeoPointField', 'PointField', 'LineStringField', 'PolygonField',
@@ -1275,6 +1274,12 @@ class GenericReferenceField(BaseField):
     """A reference to *any* :class:`~mongoengine.document.Document` subclass
     that will be automatically dereferenced on access (lazily).
 
+    Note this field works the same way as :class:`~mongoengine.document.ReferenceField`,
+    doing database I/O access the first time it is accessed (even if it's to access
+    it ``pk`` or ``id`` field).
+    To solve this you should consider using the
+    :class:`~mongoengine.fields.GenericLazyReferenceField`.
+
     .. note ::
         * Any documents used as a generic reference must be registered in the
           document registry.  Importing the model will automatically register
@@ -2159,6 +2164,8 @@ class LazyReferenceField(BaseField):
     """A really lazy reference to a document.
     Unlike the :class:`~mongoengine.fields.ReferenceField` it must be manually
     dereferenced using it ``fetch()`` method.
+
+    .. versionadded:: 0.15
     """
 
     def __init__(self, document_type, passthrough=False, dbref=False,
@@ -2274,3 +2281,65 @@ class LazyReferenceField(BaseField):
 
     def lookup_member(self, member_name):
         return self.document_type._fields.get(member_name)
+
+
+class GenericLazyReferenceField(GenericReferenceField):
+    """A reference to *any* :class:`~mongoengine.document.Document` subclass
+    that will be automatically dereferenced on access (lazily).
+    Unlike the :class:`~mongoengine.fields.GenericReferenceField` it must be
+    manually dereferenced using it ``fetch()`` method.
+
+    .. note ::
+        * Any documents used as a generic reference must be registered in the
+          document registry.  Importing the model will automatically register
+          it.
+
+        * You can use the choices param to limit the acceptable Document types
+
+    .. versionadded:: 0.15
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.passthrough = kwargs.pop('passthrough', False)
+        super(GenericLazyReferenceField, self).__init__(*args, **kwargs)
+
+    def _validate_choices(self, value):
+        if isinstance(value, LazyReference):
+            value = value.document_type
+        super(GenericLazyReferenceField, self)._validate_choices(value)
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+
+        value = instance._data.get(self.name)
+        if isinstance(value, LazyReference):
+            if value.passthrough != self.passthrough:
+                instance._data[self.name] = LazyReference(
+                    value.document_type, value.pk, passthrough=self.passthrough)
+        elif value is not None:
+            if isinstance(value, (dict, SON)):
+                value = LazyReference(get_document(value['_cls']), value['_ref'].id, passthrough=self.passthrough)
+            elif isinstance(value, Document):
+                value = LazyReference(type(value), value.pk, passthrough=self.passthrough)
+            instance._data[self.name] = value
+
+        return super(GenericLazyReferenceField, self).__get__(instance, owner)
+
+    def validate(self, value):
+        if isinstance(value, LazyReference) and value.pk is None:
+            self.error('You can only reference documents once they have been'
+                       ' saved to the database')
+        return super(GenericLazyReferenceField, self).validate(value)
+
+    def to_mongo(self, document):
+        if document is None:
+            return None
+
+        if isinstance(document, LazyReference):
+            return SON((
+                ('_cls', document.document_type._class_name),
+                ('_ref', document)
+            ))
+        else:
+            return super(GenericLazyReferenceField, self).to_mongo(document)
diff --git a/tests/fields/fields.py b/tests/fields/fields.py
index 84156622..632f5404 100644
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -933,7 +933,7 @@ class FieldTest(MongoDBTestCase):
             authors = ListField(ReferenceField(User))
             authors_as_lazy = ListField(LazyReferenceField(User))
             generic = ListField(GenericReferenceField())
-            # generic_as_lazy = ListField(LazyGenericReferenceField())
+            generic_as_lazy = ListField(GenericLazyReferenceField())
 
         User.drop_collection()
         BlogPost.drop_collection()
@@ -992,17 +992,17 @@ class FieldTest(MongoDBTestCase):
         post.generic = [user]
         post.validate()
 
-        # post.generic_as_lazy = [1, 2]
-        # self.assertRaises(ValidationError, post.validate)
+        post.generic_as_lazy = [1, 2]
+        self.assertRaises(ValidationError, post.validate)
 
-        # post.generic_as_lazy = [User(), Comment()]
-        # self.assertRaises(ValidationError, post.validate)
+        post.generic_as_lazy = [User(), Comment()]
+        self.assertRaises(ValidationError, post.validate)
 
-        # post.generic_as_lazy = [Comment()]
-        # self.assertRaises(ValidationError, post.validate)
+        post.generic_as_lazy = [Comment()]
+        self.assertRaises(ValidationError, post.validate)
 
-        # post.generic_as_lazy = [user]
-        # post.validate()
+        post.generic_as_lazy = [user]
+        post.validate()
 
     def test_sorted_list_sorting(self):
         """Ensure that a sorted list field properly sorts values.
@@ -4872,5 +4872,185 @@ class LazyReferenceFieldTest(MongoDBTestCase):
         self.assertNotEqual(other_animalref, animal)
 
 
+class GenericLazyReferenceFieldTest(MongoDBTestCase):
+    def test_generic_lazy_reference_simple(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            person = StringField()
+            animal = GenericLazyReferenceField()
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal = Animal(name="Leopard", tag="heavy").save()
+        Ocurrence(person="test", animal=animal).save()
+        p = Ocurrence.objects.get()
+        self.assertIsInstance(p.animal, LazyReference)
+        fetched_animal = p.animal.fetch()
+        self.assertEqual(fetched_animal, animal)
+        # `fetch` keep cache on referenced document by default...
+        animal.tag = "not so heavy"
+        animal.save()
+        double_fetch = p.animal.fetch()
+        self.assertIs(fetched_animal, double_fetch)
+        self.assertEqual(double_fetch.tag, "heavy")
+        # ...unless specified otherwise
+        fetch_force = p.animal.fetch(force=True)
+        self.assertIsNot(fetch_force, fetched_animal)
+        self.assertEqual(fetch_force.tag, "not so heavy")
+
+    def test_generic_lazy_reference_choices(self):
+        class Animal(Document):
+            name = StringField()
+
+        class Vegetal(Document):
+            name = StringField()
+
+        class Mineral(Document):
+            name = StringField()
+
+        class Ocurrence(Document):
+            living_thing = GenericLazyReferenceField(choices=[Animal, Vegetal])
+            thing = GenericLazyReferenceField()
+
+        Animal.drop_collection()
+        Vegetal.drop_collection()
+        Mineral.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal = Animal(name="Leopard").save()
+        vegetal = Vegetal(name="Oak").save()
+        mineral = Mineral(name="Granite").save()
+
+        occ_animal = Ocurrence(living_thing=animal, thing=animal).save()
+        occ_vegetal = Ocurrence(living_thing=vegetal, thing=vegetal).save()
+        with self.assertRaises(ValidationError):
+            Ocurrence(living_thing=mineral).save()
+
+        occ = Ocurrence.objects.get(living_thing=animal)
+        self.assertEqual(occ, occ_animal)
+        self.assertIsInstance(occ.thing, LazyReference)
+        self.assertIsInstance(occ.living_thing, LazyReference)
+
+        occ.thing = vegetal
+        occ.living_thing = vegetal
+        occ.save()
+
+        occ.thing = mineral
+        occ.living_thing = mineral
+        with self.assertRaises(ValidationError):
+            occ.save()
+
+    def test_generic_lazy_reference_set(self):
+        class Animal(Document):
+            meta = {'allow_inheritance': True}
+
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            person = StringField()
+            animal = GenericLazyReferenceField()
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        class SubAnimal(Animal):
+            nick = StringField()
+
+        animal = Animal(name="Leopard", tag="heavy").save()
+        sub_animal = SubAnimal(nick='doggo', name='dog').save()
+        for ref in (
+                animal,
+                LazyReference(Animal, animal.pk),
+                {'_cls': 'Animal', '_ref': DBRef(animal._get_collection_name(), animal.pk)},
+
+                sub_animal,
+                LazyReference(SubAnimal, sub_animal.pk),
+                {'_cls': 'SubAnimal', '_ref': DBRef(sub_animal._get_collection_name(), sub_animal.pk)},
+                ):
+            p = Ocurrence(person="test", animal=ref).save()
+            p.reload()
+            self.assertIsInstance(p.animal, (LazyReference, Document))
+            p.animal.fetch()
+
+    def test_generic_lazy_reference_bad_set(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            person = StringField()
+            animal = GenericLazyReferenceField(choices=['Animal'])
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        class BadDoc(Document):
+            pass
+
+        animal = Animal(name="Leopard", tag="heavy").save()
+        baddoc = BadDoc().save()
+        for bad in (
+                42,
+                'foo',
+                baddoc,
+                LazyReference(BadDoc, animal.pk)
+                ):
+            with self.assertRaises(ValidationError):
+                p = Ocurrence(person="test", animal=bad).save()
+
+    def test_generic_lazy_reference_query_conversion(self):
+        class Member(Document):
+            user_num = IntField(primary_key=True)
+
+        class BlogPost(Document):
+            title = StringField()
+            author = GenericLazyReferenceField()
+
+        Member.drop_collection()
+        BlogPost.drop_collection()
+
+        m1 = Member(user_num=1)
+        m1.save()
+        m2 = Member(user_num=2)
+        m2.save()
+
+        post1 = BlogPost(title='post 1', author=m1)
+        post1.save()
+
+        post2 = BlogPost(title='post 2', author=m2)
+        post2.save()
+
+        post = BlogPost.objects(author=m1).first()
+        self.assertEqual(post.id, post1.id)
+
+        post = BlogPost.objects(author=m2).first()
+        self.assertEqual(post.id, post2.id)
+
+        # Same thing by passing a LazyReference instance
+        post = BlogPost.objects(author=LazyReference(Member, m2.pk)).first()
+        self.assertEqual(post.id, post2.id)
+
+    def test_generic_lazy_reference_not_set(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            person = StringField()
+            animal = GenericLazyReferenceField()
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        Ocurrence(person='foo').save()
+        p = Ocurrence.objects.get()
+        self.assertIs(p.animal, None)
+
+
 if __name__ == '__main__':
     unittest.main()

From da33cb54fe3e80490e55957da914fec6bbde92ba Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Mon, 6 Nov 2017 14:11:11 +0100
Subject: [PATCH 037/121] Correct style

---
 mongoengine/base/datastructures.py |  2 +-
 mongoengine/fields.py              | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py
index 043df471..43f32810 100644
--- a/mongoengine/base/datastructures.py
+++ b/mongoengine/base/datastructures.py
@@ -1,8 +1,8 @@
 import itertools
 import weakref
 
-import six
 from bson import DBRef
+import six
 
 from mongoengine.common import _import_class
 from mongoengine.errors import DoesNotExist, MultipleObjectsReturned
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 61e0cb69..62d9b941 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -26,8 +26,8 @@ except ImportError:
     Int64 = long
 
 from mongoengine.base import (BaseDocument, BaseField, ComplexBaseField,
-                              GeoJsonBaseField, ObjectIdField, get_document,
-                              LazyReference)
+                              GeoJsonBaseField, LazyReference, ObjectIdField,
+                              get_document)
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
@@ -2265,9 +2265,10 @@ class LazyReferenceField(BaseField):
             try:
                 id_field.validate(pk)
             except ValidationError:
-                self.error("value should be `{0}` document, LazyReference or DBRef on `{0}` "
-                           "or `{0}`'s primary key (i.e. `{1}`)".format(
-                            self.document_type.__name__, type(id_field).__name__))
+                self.error(
+                    "value should be `{0}` document, LazyReference or DBRef on `{0}` "
+                    "or `{0}`'s primary key (i.e. `{1}`)".format(
+                        self.document_type.__name__, type(id_field).__name__))
 
         if pk is None:
             self.error('You can only reference documents once they have been '

From ea25972257fd75ddf6a5cee674e1b9ed67f80ad7 Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Mon, 6 Nov 2017 14:39:10 +0100
Subject: [PATCH 038/121] Bump version to 0.15.0

---
 mongoengine/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py
index c7c6f707..a1b7d682 100644
--- a/mongoengine/__init__.py
+++ b/mongoengine/__init__.py
@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
            list(signals.__all__) + list(errors.__all__))
 
 
-VERSION = (0, 14, 3)
+VERSION = (0, 15, 0)
 
 
 def get_version():

From 4f5b0634ad944534012a7752f26dfbe541160a2c Mon Sep 17 00:00:00 2001
From: Ryan Scott <ryan@strongarm.io>
Date: Tue, 7 Nov 2017 16:26:01 -0500
Subject: [PATCH 039/121] Add documentation for auto_create_index to the
 Indexing section of the Docs

---
 docs/guide/defining-documents.rst | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst
index d41ae7e6..5ff7a56b 100644
--- a/docs/guide/defining-documents.rst
+++ b/docs/guide/defining-documents.rst
@@ -526,8 +526,9 @@ There are a few top level defaults for all indexes that can be set::
         meta = {
             'index_options': {},
             'index_background': True,
+            'index_cls': False,
+            'auto_create_index': True,
             'index_drop_dups': True,
-            'index_cls': False
         }
 
 
@@ -540,6 +541,12 @@ There are a few top level defaults for all indexes that can be set::
 :attr:`index_cls` (Optional)
     A way to turn off a specific index for _cls.
 
+:attr:`auto_create_index` (Optional)
+    When this is True (default), MongoEngine will ensure that the correct
+    indexes exist in MongoDB each time a command is run. This can be disabled
+    in systems where indexes are managed separately. Disabling this will improve
+    performance.
+
 :attr:`index_drop_dups` (Optional)
     Set the default value for if an index should drop duplicates
 

From 47c7cb932741098e264dc9000e1ddf4fe3ed76fb Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Wed, 15 Nov 2017 20:44:36 +0100
Subject: [PATCH 040/121] Ignore style I202 rule (see
 https://github.com/PyCQA/flake8-import-order/issues/122)

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index eabe3271..46edff3b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -5,7 +5,7 @@ tests=tests
 cover-package=mongoengine
 
 [flake8]
-ignore=E501,F401,F403,F405,I201
+ignore=E501,F401,F403,F405,I201,I202
 exclude=build,dist,docs,venv,venv3,.tox,.eggs,tests
 max-complexity=47
 application-import-names=mongoengine,tests

From c1c09fa6b4c832b99afeea83d19d290468689fe4 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Tue, 21 Nov 2017 21:56:26 +0800
Subject: [PATCH 041/121] fix validatione error for invalid embedded document
 instance #1067

---
 mongoengine/fields.py      |  6 +++++-
 tests/queryset/queryset.py | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 6c4a06c9..0aa51b2d 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -688,6 +688,11 @@ class GenericEmbeddedDocumentField(BaseField):
         return value
 
     def validate(self, value, clean=True):
+        if self.choices and isinstance(value, SON):
+            for choice in self.choices:
+                if value['_cls'] == choice._class_name:
+                    return True
+
         if not isinstance(value, EmbeddedDocument):
             self.error('Invalid embedded document instance provided to an '
                        'GenericEmbeddedDocumentField')
@@ -705,7 +710,6 @@ class GenericEmbeddedDocumentField(BaseField):
     def to_mongo(self, document, use_db_field=True, fields=None):
         if document is None:
             return None
-
         data = document.to_mongo(use_db_field, fields)
         if '_cls' not in data:
             data['_cls'] = document._class_name
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 28e84af4..43800fff 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -2086,6 +2086,23 @@ class QuerySetTest(unittest.TestCase):
             Site.objects(id=s.id).update_one(
                 pull_all__collaborators__helpful__user=['Ross'])
 
+    def test_pull_in_genericembedded_field(self):
+
+        class Foo(EmbeddedDocument):
+            name = StringField()
+
+        class Bar(Document):
+            foos = ListField(GenericEmbeddedDocumentField(
+                choices=[Foo, ]))
+
+        Bar.drop_collection()
+
+        foo = Foo(name="bar")
+        bar = Bar(foos=[foo]).save()
+        Bar.objects(id=bar.id).update(pull__foos=foo)
+        bar.reload()
+        self.assertEqual(len(bar.foos), 0)
+
     def test_update_one_pop_generic_reference(self):
 
         class BlogTag(Document):
@@ -2179,6 +2196,24 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(message.authors[1].name, "Ross")
         self.assertEqual(message.authors[2].name, "Adam")
 
+    def test_set_generic_embedded_documents(self):
+
+        class Bar(EmbeddedDocument):
+            name = StringField()
+
+        class User(Document):
+            username = StringField()
+            bar = GenericEmbeddedDocumentField(choices=[Bar,])
+
+        User.drop_collection()
+
+        User(username='abc').save()
+        User.objects(username='abc').update(
+            set__bar=Bar(name='test'), upsert=True)
+
+        user = User.objects(username='abc').first()
+        self.assertEqual(user.bar.name, "test")
+
     def test_reload_embedded_docs_instance(self):
 
         class SubDoc(EmbeddedDocument):

From e74f65901565f64b39a51edc22a1564d8f9fed9a Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Wed, 8 Nov 2017 18:05:30 +0100
Subject: [PATCH 042/121] Improve LazyReferenceField and
 GenericLazyReferenceField with nested fields

---
 mongoengine/base/document.py |  3 +-
 mongoengine/dereference.py   |  9 +++-
 mongoengine/fields.py        | 56 +++++++++++++++--------
 setup.cfg                    |  2 +-
 tests/fields/fields.py       | 86 ++++++++++++++++++++++++++++++++++++
 5 files changed, 134 insertions(+), 22 deletions(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index f8ab73d0..658d0c79 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -13,6 +13,7 @@ from mongoengine import signals
 from mongoengine.base.common import get_document
 from mongoengine.base.datastructures import (BaseDict, BaseList,
                                              EmbeddedDocumentList,
+                                             LazyReference,
                                              StrictDict)
 from mongoengine.base.fields import ComplexBaseField
 from mongoengine.common import _import_class
@@ -488,7 +489,7 @@ class BaseDocument(object):
                 else:
                     data = getattr(data, part, None)
 
-                if hasattr(data, '_changed_fields'):
+                if not isinstance(data, LazyReference) and hasattr(data, '_changed_fields'):
                     if getattr(data, '_is_document', False):
                         continue
 
diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py
index 59204d4d..7fe34e43 100644
--- a/mongoengine/dereference.py
+++ b/mongoengine/dereference.py
@@ -3,6 +3,7 @@ import six
 
 from mongoengine.base import (BaseDict, BaseList, EmbeddedDocumentList,
                               TopLevelDocumentMetaclass, get_document)
+from mongoengine.base.datastructures import LazyReference
 from mongoengine.connection import get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.fields import DictField, ListField, MapField, ReferenceField
@@ -99,7 +100,10 @@ class DeReference(object):
             if isinstance(item, (Document, EmbeddedDocument)):
                 for field_name, field in item._fields.iteritems():
                     v = item._data.get(field_name, None)
-                    if isinstance(v, DBRef):
+                    if isinstance(v, LazyReference):
+                        # LazyReference inherits DBRef but should not be dereferenced here !
+                        continue
+                    elif isinstance(v, DBRef):
                         reference_map.setdefault(field.document_type, set()).add(v.id)
                     elif isinstance(v, (dict, SON)) and '_ref' in v:
                         reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)
@@ -110,6 +114,9 @@ class DeReference(object):
                             if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):
                                 key = field_cls
                             reference_map.setdefault(key, set()).update(refs)
+            elif isinstance(item, LazyReference):
+                # LazyReference inherits DBRef but should not be dereferenced here !
+                continue
             elif isinstance(item, DBRef):
                 reference_map.setdefault(item.collection, set()).add(item.id)
             elif isinstance(item, (dict, SON)) and '_ref' in item:
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 6c4a06c9..8ca2b17f 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -28,6 +28,7 @@ except ImportError:
 from mongoengine.base import (BaseDocument, BaseField, ComplexBaseField,
                               GeoJsonBaseField, LazyReference, ObjectIdField,
                               get_document)
+from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
@@ -789,6 +790,17 @@ class ListField(ComplexBaseField):
         kwargs.setdefault('default', lambda: [])
         super(ListField, self).__init__(**kwargs)
 
+    def __get__(self, instance, owner):
+        if instance is None:
+            # Document class being used rather than a document object
+            return self
+        value = instance._data.get(self.name)
+        LazyReferenceField = _import_class('LazyReferenceField')
+        GenericLazyReferenceField = _import_class('GenericLazyReferenceField')
+        if isinstance(self.field, (LazyReferenceField, GenericLazyReferenceField)) and value:
+            instance._data[self.name] = [self.field.build_lazyref(x) for x in value]
+        return super(ListField, self).__get__(instance, owner)
+
     def validate(self, value):
         """Make sure that a list of valid fields is being used."""
         if (not isinstance(value, (list, tuple, QuerySet)) or
@@ -2211,17 +2223,10 @@ class LazyReferenceField(BaseField):
                 self.document_type_obj = get_document(self.document_type_obj)
         return self.document_type_obj
 
-    def __get__(self, instance, owner):
-        """Descriptor to allow lazy dereferencing."""
-        if instance is None:
-            # Document class being used rather than a document object
-            return self
-
-        value = instance._data.get(self.name)
+    def build_lazyref(self, value):
         if isinstance(value, LazyReference):
             if value.passthrough != self.passthrough:
-                instance._data[self.name] = LazyReference(
-                    value.document_type, value.pk, passthrough=self.passthrough)
+                value = LazyReference(value.document_type, value.pk, passthrough=self.passthrough)
         elif value is not None:
             if isinstance(value, self.document_type):
                 value = LazyReference(self.document_type, value.pk, passthrough=self.passthrough)
@@ -2230,6 +2235,16 @@ class LazyReferenceField(BaseField):
             else:
                 # value is the primary key of the referenced document
                 value = LazyReference(self.document_type, value, passthrough=self.passthrough)
+        return value
+
+    def __get__(self, instance, owner):
+        """Descriptor to allow lazy dereferencing."""
+        if instance is None:
+            # Document class being used rather than a document object
+            return self
+
+        value = self.build_lazyref(instance._data.get(self.name))
+        if value:
             instance._data[self.name] = value
 
         return super(LazyReferenceField, self).__get__(instance, owner)
@@ -2254,7 +2269,7 @@ class LazyReferenceField(BaseField):
 
     def validate(self, value):
         if isinstance(value, LazyReference):
-            if not issubclass(value.document_type, self.document_type):
+            if value.collection != self.document_type._get_collection_name():
                 self.error('Reference must be on a `%s` document.' % self.document_type)
             pk = value.pk
         elif isinstance(value, self.document_type):
@@ -2314,23 +2329,26 @@ class GenericLazyReferenceField(GenericReferenceField):
 
     def _validate_choices(self, value):
         if isinstance(value, LazyReference):
-            value = value.document_type
+            value = value.document_type._class_name
         super(GenericLazyReferenceField, self)._validate_choices(value)
 
-    def __get__(self, instance, owner):
-        if instance is None:
-            return self
-
-        value = instance._data.get(self.name)
+    def build_lazyref(self, value):
         if isinstance(value, LazyReference):
             if value.passthrough != self.passthrough:
-                instance._data[self.name] = LazyReference(
-                    value.document_type, value.pk, passthrough=self.passthrough)
+                value = LazyReference(value.document_type, value.pk, passthrough=self.passthrough)
         elif value is not None:
             if isinstance(value, (dict, SON)):
                 value = LazyReference(get_document(value['_cls']), value['_ref'].id, passthrough=self.passthrough)
             elif isinstance(value, Document):
                 value = LazyReference(type(value), value.pk, passthrough=self.passthrough)
+        return value
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+
+        value = self.build_lazyref(instance._data.get(self.name))
+        if value:
             instance._data[self.name] = value
 
         return super(GenericLazyReferenceField, self).__get__(instance, owner)
@@ -2348,7 +2366,7 @@ class GenericLazyReferenceField(GenericReferenceField):
         if isinstance(document, LazyReference):
             return SON((
                 ('_cls', document.document_type._class_name),
-                ('_ref', document)
+                ('_ref', DBRef(document.document_type._get_collection_name(), document.pk))
             ))
         else:
             return super(GenericLazyReferenceField, self).to_mongo(document)
diff --git a/setup.cfg b/setup.cfg
index 46edff3b..fd6192b8 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,7 +1,7 @@
 [nosetests]
 verbosity=2
 detailed-errors=1
-tests=tests
+#tests=tests
 cover-package=mongoengine
 
 [flake8]
diff --git a/tests/fields/fields.py b/tests/fields/fields.py
index 632f5404..ffee25e6 100644
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -4871,6 +4871,48 @@ class LazyReferenceFieldTest(MongoDBTestCase):
         self.assertNotEqual(animal, other_animalref)
         self.assertNotEqual(other_animalref, animal)
 
+    def test_lazy_reference_embedded(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class EmbeddedOcurrence(EmbeddedDocument):
+            in_list = ListField(LazyReferenceField(Animal))
+            direct = LazyReferenceField(Animal)
+
+        class Ocurrence(Document):
+            in_list = ListField(LazyReferenceField(Animal))
+            in_embedded = EmbeddedDocumentField(EmbeddedOcurrence)
+            direct = LazyReferenceField(Animal)
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal1 = Animal('doggo').save()
+        animal2 = Animal('cheeta').save()
+
+        def check_fields_type(occ):
+            self.assertIsInstance(occ.direct, LazyReference)
+            for elem in occ.in_list:
+                self.assertIsInstance(elem, LazyReference)
+            self.assertIsInstance(occ.in_embedded.direct, LazyReference)
+            for elem in occ.in_embedded.in_list:
+                self.assertIsInstance(elem, LazyReference)
+
+        occ = Ocurrence(
+            in_list=[animal1, animal2],
+            in_embedded={'in_list': [animal1, animal2], 'direct': animal1},
+            direct=animal1
+        ).save()
+        check_fields_type(occ)
+        occ.reload()
+        check_fields_type(occ)
+        occ.direct = animal1.id
+        occ.in_list = [animal1.id, animal2.id]
+        occ.in_embedded.direct = animal1.id
+        occ.in_embedded.in_list = [animal1.id, animal2.id]
+        check_fields_type(occ)
+
 
 class GenericLazyReferenceFieldTest(MongoDBTestCase):
     def test_generic_lazy_reference_simple(self):
@@ -5051,6 +5093,50 @@ class GenericLazyReferenceFieldTest(MongoDBTestCase):
         p = Ocurrence.objects.get()
         self.assertIs(p.animal, None)
 
+    def test_generic_lazy_reference_embedded(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class EmbeddedOcurrence(EmbeddedDocument):
+            in_list = ListField(GenericLazyReferenceField())
+            direct = GenericLazyReferenceField()
+
+        class Ocurrence(Document):
+            in_list = ListField(GenericLazyReferenceField())
+            in_embedded = EmbeddedDocumentField(EmbeddedOcurrence)
+            direct = GenericLazyReferenceField()
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal1 = Animal('doggo').save()
+        animal2 = Animal('cheeta').save()
+
+        def check_fields_type(occ):
+            self.assertIsInstance(occ.direct, LazyReference)
+            for elem in occ.in_list:
+                self.assertIsInstance(elem, LazyReference)
+            self.assertIsInstance(occ.in_embedded.direct, LazyReference)
+            for elem in occ.in_embedded.in_list:
+                self.assertIsInstance(elem, LazyReference)
+
+        occ = Ocurrence(
+            in_list=[animal1, animal2],
+            in_embedded={'in_list': [animal1, animal2], 'direct': animal1},
+            direct=animal1
+        ).save()
+        check_fields_type(occ)
+        occ.reload()
+        check_fields_type(occ)
+        animal1_ref = {'_cls': 'Animal', '_ref': DBRef(animal1._get_collection_name(), animal1.pk)}
+        animal2_ref = {'_cls': 'Animal', '_ref': DBRef(animal2._get_collection_name(), animal2.pk)}
+        occ.direct = animal1_ref
+        occ.in_list = [animal1_ref, animal2_ref]
+        occ.in_embedded.direct = animal1_ref
+        occ.in_embedded.in_list = [animal1_ref, animal2_ref]
+        check_fields_type(occ)
+
 
 if __name__ == '__main__':
     unittest.main()

From 60758dd76b4ae1831e4f3f389c71b95b0e77abd2 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Wed, 22 Nov 2017 18:56:12 +0800
Subject: [PATCH 043/121] add changelog

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 1e9ac7fc..b4852178 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -5,6 +5,7 @@ Changelog
 Changes in 0.15.0
 =================
 - Add LazyReferenceField and GenericLazyReferenceField to address #1230
+- Fix validation error for invalid embedded document instance #1067
 
 Changes in 0.14.1
 =================

From 79486e33936d6302e1cb898c38a9535648563343 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Wed, 22 Nov 2017 19:27:35 +0800
Subject: [PATCH 044/121] change description in changelog

---
 docs/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index b4852178..249d4659 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -5,7 +5,7 @@ Changelog
 Changes in 0.15.0
 =================
 - Add LazyReferenceField and GenericLazyReferenceField to address #1230
-- Fix validation error for invalid embedded document instance #1067
+- Fix validation error instance in GenericEmbeddedDocumentField #1067
 
 Changes in 0.14.1
 =================

From 0ce081323fd66ad00b00eb4bc9edc6b888a5fa2d Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Wed, 22 Nov 2017 22:19:50 +0800
Subject: [PATCH 045/121] move changes to development

---
 docs/changelog.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 249d4659..f96fc11b 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -2,10 +2,13 @@
 Changelog
 =========
 
+Development
+===========
+- Fix validation error instance in GenericEmbeddedDocumentField #1067
+
 Changes in 0.15.0
 =================
 - Add LazyReferenceField and GenericLazyReferenceField to address #1230
-- Fix validation error instance in GenericEmbeddedDocumentField #1067
 
 Changes in 0.14.1
 =================

From c45dfacb411a062dc75acd27e784693fb6c0898a Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Wed, 29 Nov 2017 16:41:42 +0100
Subject: [PATCH 046/121] Update changelog

---
 docs/changelog.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 1e9ac7fc..151fdd08 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -2,6 +2,13 @@
 Changelog
 =========
 
+dev
+===
+-  Subfield resolve error in generic_emdedded_document query #1651 #1652
+-  use each modifier only with $position #1673 #1675
+-  Improve LazyReferenceField and GenericLazyReferenceField with nested fields #1704
+
+
 Changes in 0.15.0
 =================
 - Add LazyReferenceField and GenericLazyReferenceField to address #1230

From b35efb9f7205ab9b07d5b21c5cdb26d4c6d07201 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Tue, 21 Nov 2017 21:56:26 +0800
Subject: [PATCH 047/121] fix validatione error for invalid embedded document
 instance #1067

---
 mongoengine/fields.py      |  6 +++++-
 tests/queryset/queryset.py | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 8ca2b17f..4e941c4e 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -689,6 +689,11 @@ class GenericEmbeddedDocumentField(BaseField):
         return value
 
     def validate(self, value, clean=True):
+        if self.choices and isinstance(value, SON):
+            for choice in self.choices:
+                if value['_cls'] == choice._class_name:
+                    return True
+
         if not isinstance(value, EmbeddedDocument):
             self.error('Invalid embedded document instance provided to an '
                        'GenericEmbeddedDocumentField')
@@ -706,7 +711,6 @@ class GenericEmbeddedDocumentField(BaseField):
     def to_mongo(self, document, use_db_field=True, fields=None):
         if document is None:
             return None
-
         data = document.to_mongo(use_db_field, fields)
         if '_cls' not in data:
             data['_cls'] = document._class_name
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 28e84af4..43800fff 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -2086,6 +2086,23 @@ class QuerySetTest(unittest.TestCase):
             Site.objects(id=s.id).update_one(
                 pull_all__collaborators__helpful__user=['Ross'])
 
+    def test_pull_in_genericembedded_field(self):
+
+        class Foo(EmbeddedDocument):
+            name = StringField()
+
+        class Bar(Document):
+            foos = ListField(GenericEmbeddedDocumentField(
+                choices=[Foo, ]))
+
+        Bar.drop_collection()
+
+        foo = Foo(name="bar")
+        bar = Bar(foos=[foo]).save()
+        Bar.objects(id=bar.id).update(pull__foos=foo)
+        bar.reload()
+        self.assertEqual(len(bar.foos), 0)
+
     def test_update_one_pop_generic_reference(self):
 
         class BlogTag(Document):
@@ -2179,6 +2196,24 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(message.authors[1].name, "Ross")
         self.assertEqual(message.authors[2].name, "Adam")
 
+    def test_set_generic_embedded_documents(self):
+
+        class Bar(EmbeddedDocument):
+            name = StringField()
+
+        class User(Document):
+            username = StringField()
+            bar = GenericEmbeddedDocumentField(choices=[Bar,])
+
+        User.drop_collection()
+
+        User(username='abc').save()
+        User.objects(username='abc').update(
+            set__bar=Bar(name='test'), upsert=True)
+
+        user = User.objects(username='abc').first()
+        self.assertEqual(user.bar.name, "test")
+
     def test_reload_embedded_docs_instance(self):
 
         class SubDoc(EmbeddedDocument):

From ce9ea7baad2870145203a51e069f6a6f57526518 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Wed, 22 Nov 2017 18:56:12 +0800
Subject: [PATCH 048/121] add changelog

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 151fdd08..69e57e0b 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -12,6 +12,7 @@ dev
 Changes in 0.15.0
 =================
 - Add LazyReferenceField and GenericLazyReferenceField to address #1230
+- Fix validation error for invalid embedded document instance #1067
 
 Changes in 0.14.1
 =================

From 08a4deca17b511974125e26a7151e165231b58de Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Wed, 22 Nov 2017 19:27:35 +0800
Subject: [PATCH 049/121] change description in changelog

---
 docs/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 69e57e0b..c44f8432 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -12,7 +12,7 @@ dev
 Changes in 0.15.0
 =================
 - Add LazyReferenceField and GenericLazyReferenceField to address #1230
-- Fix validation error for invalid embedded document instance #1067
+- Fix validation error instance in GenericEmbeddedDocumentField #1067
 
 Changes in 0.14.1
 =================

From 2cbebf9c99dd5ebd368ca775b60fc25149ea5634 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul@shine.mn>
Date: Wed, 22 Nov 2017 22:19:50 +0800
Subject: [PATCH 050/121] move changes to development

---
 docs/changelog.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index c44f8432..efd2e0f6 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -7,12 +7,11 @@ dev
 -  Subfield resolve error in generic_emdedded_document query #1651 #1652
 -  use each modifier only with $position #1673 #1675
 -  Improve LazyReferenceField and GenericLazyReferenceField with nested fields #1704
-
+-  Fix validation error instance in GenericEmbeddedDocumentField #1067
 
 Changes in 0.15.0
 =================
 - Add LazyReferenceField and GenericLazyReferenceField to address #1230
-- Fix validation error instance in GenericEmbeddedDocumentField #1067
 
 Changes in 0.14.1
 =================

From 9e0ca51c2f47618b539c2724ecfe656e454eb062 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Fri, 1 Dec 2017 08:40:51 +0800
Subject: [PATCH 051/121] remove merge conflict after rebase #1067

---
 docs/changelog.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 1bd932d6..efd2e0f6 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -2,7 +2,6 @@
 Changelog
 =========
 
-<<<<<<< HEAD
 dev
 ===
 -  Subfield resolve error in generic_emdedded_document query #1651 #1652

From 7674dc9b344f99e47a6df3b1c44d72faf57ff137 Mon Sep 17 00:00:00 2001
From: Esmail <esmail@users.noreply.github.com>
Date: Tue, 5 Dec 2017 15:14:15 -0500
Subject: [PATCH 052/121] One-character typo fix ("that" -> "than")

---
 docs/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index cc5b647d..ea1a04c1 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -153,7 +153,7 @@ post. This works, but there is no real reason to be storing the comments
 separately from their associated posts, other than to work around the
 relational model. Using MongoDB we can store the comments as a list of
 *embedded documents* directly on a post document. An embedded document should
-be treated no differently that a regular document; it just doesn't have its own
+be treated no differently than a regular document; it just doesn't have its own
 collection in the database. Using MongoEngine, we can define the structure of
 embedded documents, along with utility methods, in exactly the same way we do
 with regular documents::

From 22a8ad2fde14b0c5eb2cf3cbc0fd4e0de98d67c5 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Wed, 20 Dec 2017 15:17:54 +0800
Subject: [PATCH 053/121] update fields argument when given #1172

---
 mongoengine/fields.py  |  6 +++++-
 tests/fields/fields.py | 45 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 8ca2b17f..17003974 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -1256,7 +1256,11 @@ class CachedReferenceField(BaseField):
             if value.pk is None:
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
-            return {'_id': value.pk}
+            value_dict = {'_id': value.pk}
+            for field in self.fields:
+                value_dict.update({field: value[field]})
+
+            return value_dict
 
         raise NotImplementedError
 
diff --git a/tests/fields/fields.py b/tests/fields/fields.py
index ffee25e6..f86ffdb4 100644
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -4379,6 +4379,51 @@ class CachedReferenceFieldTest(MongoDBTestCase):
         self.assertEqual(SocialData.objects(person__group=g2).count(), 1)
         self.assertEqual(SocialData.objects(person__group=g2).first(), s2)
 
+    def test_cached_reference_field_push_with_fields(self):
+        class Product(Document):
+            name = StringField()
+
+        Product.drop_collection()
+
+        class Basket(Document):
+            products = ListField(CachedReferenceField(Product, fields=['name']))
+
+        Basket.drop_collection()
+        product1 = Product(name='abc').save()
+        product2 = Product(name='def').save()
+        basket = Basket(products=[product1]).save()
+        self.assertEqual(
+            Basket.objects._collection.find_one(),
+            {
+                '_id': basket.pk,
+                'products': [
+                    {
+                        '_id': product1.pk,
+                        'name': product1.name
+                    }
+                ]
+            }
+        )
+        # push to list
+        basket.update(push__products=product2)
+        basket.reload()
+        self.assertEqual(
+            Basket.objects._collection.find_one(),
+            {
+                '_id': basket.pk,
+                'products': [
+                    {
+                        '_id': product1.pk,
+                        'name': product1.name
+                    },
+                    {
+                        '_id': product2.pk,
+                        'name': product2.name
+                    }
+                ]
+            }
+        )
+
     def test_cached_reference_field_update_all(self):
         class Person(Document):
             TYPES = (

From 6621c318db9fc5d3395d9ceb9d0900cc0fef0e68 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Wed, 20 Dec 2017 15:28:33 +0800
Subject: [PATCH 054/121] add changelog #1712

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 151fdd08..7b937822 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -7,6 +7,7 @@ dev
 -  Subfield resolve error in generic_emdedded_document query #1651 #1652
 -  use each modifier only with $position #1673 #1675
 -  Improve LazyReferenceField and GenericLazyReferenceField with nested fields #1704
+-  Update cached fields when fields argument is given #1712
 
 
 Changes in 0.15.0

From 18a5fba42b2af5e0835293b1e7201747100014c9 Mon Sep 17 00:00:00 2001
From: erdenezul <erdenezul@shine.mn>
Date: Fri, 22 Dec 2017 20:19:21 +0800
Subject: [PATCH 055/121] Revert "add tests to increase code coverage"

---
 tests/fields/fields.py     | 36 +-----------------------------------
 tests/queryset/geo.py      |  4 ----
 tests/queryset/queryset.py | 10 ----------
 3 files changed, 1 insertion(+), 49 deletions(-)

diff --git a/tests/fields/fields.py b/tests/fields/fields.py
index 4360f298..ffee25e6 100644
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -312,27 +312,6 @@ class FieldTest(MongoDBTestCase):
 
         self.assertEqual(1, TestDocument.objects(long_fld__ne=None).count())
 
-    def test_callable_validation(self):
-        """Ensure that callable validation works"""
-        def check_even(value):
-            return value % 2 == 0
-
-        class Order(Document):
-            number = IntField(validation=check_even)
-
-        Order.drop_collection()
-
-        order = Order(number=3)
-        self.assertRaises(ValidationError, order.validate)
-
-        class User(Document):
-            name = StringField(validation=1)
-
-        User.drop_collection()
-
-        user = User(name='test')
-        self.assertRaises(ValidationError, user.validate)
-
     def test_object_id_validation(self):
         """Ensure that invalid values cannot be assigned to an
         ObjectIdField.
@@ -355,7 +334,7 @@ class FieldTest(MongoDBTestCase):
     def test_string_validation(self):
         """Ensure that invalid values cannot be assigned to string fields."""
         class Person(Document):
-            name = StringField(max_length=20, min_length=5)
+            name = StringField(max_length=20)
             userid = StringField(r'[0-9a-z_]+$')
 
         person = Person(name=34)
@@ -373,10 +352,6 @@ class FieldTest(MongoDBTestCase):
         person = Person(name='Name that is more than twenty characters')
         self.assertRaises(ValidationError, person.validate)
 
-        # Test max length validation on name
-        person = Person(name='aa')
-        self.assertRaises(ValidationError, person.validate)
-
         person.name = 'Shorter name'
         person.validate()
 
@@ -462,10 +437,6 @@ class FieldTest(MongoDBTestCase):
         doc.age = 'ten'
         self.assertRaises(ValidationError, doc.validate)
 
-        # Test max_value validation
-        doc.value = 200
-        self.assertRaises(ValidationError, doc.validate)
-
     def test_float_validation(self):
         """Ensure that invalid values cannot be assigned to float fields.
         """
@@ -548,11 +519,6 @@ class FieldTest(MongoDBTestCase):
             class User(Document):
                 name = StringField(db_field='name\0')
 
-        # db field should be a string
-        with self.assertRaises(TypeError):
-            class User(Document):
-                name = StringField(db_field=1)
-
     def test_decimal_comparison(self):
         class Person(Document):
             money = DecimalField()
diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py
index acfd9364..38c0377e 100644
--- a/tests/queryset/geo.py
+++ b/tests/queryset/geo.py
@@ -429,10 +429,6 @@ class GeoQueriesTest(MongoDBTestCase):
         roads = Road.objects.filter(line__geo_within=polygon).count()
         self.assertEqual(1, roads)
 
-        sphere = [[-1, 42,], 2]
-        roads = Road.objects.filter(line__geo_within_sphere=sphere).count()
-        self.assertEqual(0, roads)
-
         roads = Road.objects.filter(line__geo_within={"$geometry": polygon}).count()
         self.assertEqual(1, roads)
 
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 984314ed..43800fff 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -1861,10 +1861,6 @@ class QuerySetTest(unittest.TestCase):
         post = BlogPost(name="Test Post", hits=5, tags=['test'])
         post.save()
 
-        BlogPost.objects.update(hits=11)
-        post.reload()
-        self.assertEqual(post.hits, 11)
-
         BlogPost.objects.update(set__hits=10)
         post.reload()
         self.assertEqual(post.hits, 10)
@@ -1886,12 +1882,6 @@ class QuerySetTest(unittest.TestCase):
         post.reload()
         self.assertTrue('mongo' in post.tags)
 
-        # Push with arrays
-        BlogPost.objects.update(push__tags=['python', 'scala'])
-        post.reload()
-        self.assertTrue('python' in post.tags)
-        self.assertTrue('scala' in post.tags)
-
         BlogPost.objects.update_one(push_all__tags=['db', 'nosql'])
         post.reload()
         self.assertTrue('db' in post.tags and 'nosql' in post.tags)

From 12b846586c4e72d712920d32e595146911c4f9e2 Mon Sep 17 00:00:00 2001
From: Emmanuel Leblond <emmanuel.leblond@gmail.com>
Date: Fri, 22 Dec 2017 13:23:03 +0100
Subject: [PATCH 056/121] Fix travis tests with mongodb 2.4 & pymongo 3

---
 .travis.yml | 14 +++++++-------
 tox.ini     |  5 +++--
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 78a9f787..a70c711e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,7 +21,7 @@ python:
 env:
 - MONGODB=2.6 PYMONGO=2.7
 - MONGODB=2.6 PYMONGO=2.8
-- MONGODB=2.6 PYMONGO=3.0
+- MONGODB=2.6 PYMONGO=3.x
 
 matrix:
   # Finish the build as soon as one job fails
@@ -31,19 +31,19 @@ matrix:
   - python: 2.7
     env: MONGODB=2.4 PYMONGO=2.7
   - python: 2.7
-    env: MONGODB=2.4 PYMONGO=3.0
+    env: MONGODB=2.4 PYMONGO=3.5
   - python: 2.7
-    env: MONGODB=3.0 PYMONGO=3.0
+    env: MONGODB=3.0 PYMONGO=3.x
   - python: 3.5
     env: MONGODB=2.4 PYMONGO=2.7
   - python: 3.5
-    env: MONGODB=2.4 PYMONGO=3.0
+    env: MONGODB=2.4 PYMONGO=3.5
   - python: 3.5
-    env: MONGODB=3.0 PYMONGO=3.0
+    env: MONGODB=3.0 PYMONGO=3.x
   - python: 3.6
-    env: MONGODB=2.4 PYMONGO=3.0
+    env: MONGODB=2.4 PYMONGO=3.5
   - python: 3.6
-    env: MONGODB=3.0 PYMONGO=3.0
+    env: MONGODB=3.0 PYMONGO=3.x
 
 before_install:
 - bash .install_mongodb_on_travis.sh
diff --git a/tox.ini b/tox.ini
index 7f0d36e4..9bb0c5ec 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py35,pypy,pypy3}-{mg27,mg28,mg30}
+envlist = {py27,py35,pypy,pypy3}-{mg27,mg28,mg35,mg3x}
 
 [testenv]
 commands =
@@ -8,6 +8,7 @@ deps =
     nose
     mg27: PyMongo<2.8
     mg28: PyMongo>=2.8,<2.9
-    mg30: PyMongo>=3.0
+    mg35: PyMongo==3.5
+    mg3x: PyMongo>=3.0
 setenv =
     PYTHON_EGG_CACHE = {envdir}/python-eggs

From aa5510531df9eed361e34c5130bc19ce15b1185e Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Wed, 20 Dec 2017 15:17:54 +0800
Subject: [PATCH 057/121] update fields argument when given #1172

---
 mongoengine/fields.py  |  6 +++++-
 tests/fields/fields.py | 45 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 4e941c4e..7932f73a 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -1260,7 +1260,11 @@ class CachedReferenceField(BaseField):
             if value.pk is None:
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
-            return {'_id': value.pk}
+            value_dict = {'_id': value.pk}
+            for field in self.fields:
+                value_dict.update({field: value[field]})
+
+            return value_dict
 
         raise NotImplementedError
 
diff --git a/tests/fields/fields.py b/tests/fields/fields.py
index ffee25e6..f86ffdb4 100644
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -4379,6 +4379,51 @@ class CachedReferenceFieldTest(MongoDBTestCase):
         self.assertEqual(SocialData.objects(person__group=g2).count(), 1)
         self.assertEqual(SocialData.objects(person__group=g2).first(), s2)
 
+    def test_cached_reference_field_push_with_fields(self):
+        class Product(Document):
+            name = StringField()
+
+        Product.drop_collection()
+
+        class Basket(Document):
+            products = ListField(CachedReferenceField(Product, fields=['name']))
+
+        Basket.drop_collection()
+        product1 = Product(name='abc').save()
+        product2 = Product(name='def').save()
+        basket = Basket(products=[product1]).save()
+        self.assertEqual(
+            Basket.objects._collection.find_one(),
+            {
+                '_id': basket.pk,
+                'products': [
+                    {
+                        '_id': product1.pk,
+                        'name': product1.name
+                    }
+                ]
+            }
+        )
+        # push to list
+        basket.update(push__products=product2)
+        basket.reload()
+        self.assertEqual(
+            Basket.objects._collection.find_one(),
+            {
+                '_id': basket.pk,
+                'products': [
+                    {
+                        '_id': product1.pk,
+                        'name': product1.name
+                    },
+                    {
+                        '_id': product2.pk,
+                        'name': product2.name
+                    }
+                ]
+            }
+        )
+
     def test_cached_reference_field_update_all(self):
         class Person(Document):
             TYPES = (

From b66621f9c648d6fffbfd770688d918559bc6b7fb Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Wed, 20 Dec 2017 15:28:33 +0800
Subject: [PATCH 058/121] add changelog #1712

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index efd2e0f6..219d60df 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -8,6 +8,7 @@ dev
 -  use each modifier only with $position #1673 #1675
 -  Improve LazyReferenceField and GenericLazyReferenceField with nested fields #1704
 -  Fix validation error instance in GenericEmbeddedDocumentField #1067
+-  Update cached fields when fields argument is given #1712
 
 Changes in 0.15.0
 =================

From 101947da8bec04b532b55c7a252dfff5921b3f0f Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Wed, 20 Dec 2017 15:17:54 +0800
Subject: [PATCH 059/121] update fields argument when given #1172

---
 mongoengine/fields.py  |  6 +++++-
 tests/fields/fields.py | 45 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 4e941c4e..7932f73a 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -1260,7 +1260,11 @@ class CachedReferenceField(BaseField):
             if value.pk is None:
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
-            return {'_id': value.pk}
+            value_dict = {'_id': value.pk}
+            for field in self.fields:
+                value_dict.update({field: value[field]})
+
+            return value_dict
 
         raise NotImplementedError
 
diff --git a/tests/fields/fields.py b/tests/fields/fields.py
index ffee25e6..f86ffdb4 100644
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -4379,6 +4379,51 @@ class CachedReferenceFieldTest(MongoDBTestCase):
         self.assertEqual(SocialData.objects(person__group=g2).count(), 1)
         self.assertEqual(SocialData.objects(person__group=g2).first(), s2)
 
+    def test_cached_reference_field_push_with_fields(self):
+        class Product(Document):
+            name = StringField()
+
+        Product.drop_collection()
+
+        class Basket(Document):
+            products = ListField(CachedReferenceField(Product, fields=['name']))
+
+        Basket.drop_collection()
+        product1 = Product(name='abc').save()
+        product2 = Product(name='def').save()
+        basket = Basket(products=[product1]).save()
+        self.assertEqual(
+            Basket.objects._collection.find_one(),
+            {
+                '_id': basket.pk,
+                'products': [
+                    {
+                        '_id': product1.pk,
+                        'name': product1.name
+                    }
+                ]
+            }
+        )
+        # push to list
+        basket.update(push__products=product2)
+        basket.reload()
+        self.assertEqual(
+            Basket.objects._collection.find_one(),
+            {
+                '_id': basket.pk,
+                'products': [
+                    {
+                        '_id': product1.pk,
+                        'name': product1.name
+                    },
+                    {
+                        '_id': product2.pk,
+                        'name': product2.name
+                    }
+                ]
+            }
+        )
+
     def test_cached_reference_field_update_all(self):
         class Person(Document):
             TYPES = (

From 19b18d3d0ab8be238d16a2c41734b6d5dba4adb6 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Wed, 20 Dec 2017 15:28:33 +0800
Subject: [PATCH 060/121] add changelog #1712

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index efd2e0f6..219d60df 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -8,6 +8,7 @@ dev
 -  use each modifier only with $position #1673 #1675
 -  Improve LazyReferenceField and GenericLazyReferenceField with nested fields #1704
 -  Fix validation error instance in GenericEmbeddedDocumentField #1067
+-  Update cached fields when fields argument is given #1712
 
 Changes in 0.15.0
 =================

From 919f221be9738dc278f08c31c7421267bfd4f2de Mon Sep 17 00:00:00 2001
From: Chuan-Heng Hsiao <hsiao.chuanheng@gmail.com>
Date: Thu, 11 Jan 2018 07:28:25 -0500
Subject: [PATCH 061/121] defensive programming for v as an instance of DBRef
 when accessing v.collection in dereference

---
 mongoengine/dereference.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py
index 7fe34e43..18b365cc 100644
--- a/mongoengine/dereference.py
+++ b/mongoengine/dereference.py
@@ -237,7 +237,7 @@ class DeReference(object):
             elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
                 item_name = '%s.%s' % (name, k) if name else name
                 data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)
-            elif hasattr(v, 'id'):
+            elif isinstance(v, DBRef) and hasattr(v, 'id'):
                 data[k] = self.object_map.get((v.collection, v.id), v)
 
         if instance and name:

From 22e75c1691e67a487d90166c37f3cf08531226ac Mon Sep 17 00:00:00 2001
From: Ivan Pogrebkov <chezhik@gmail.com>
Date: Fri, 26 Jan 2018 10:55:44 +0300
Subject: [PATCH 062/121] Insert null values fix

https://stackoverflow.com/questions/42601950/how-to-store-a-null-value-in-mongodb-via-mongoengine
---
 mongoengine/base/document.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index 658d0c79..956e9b0e 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -337,11 +337,10 @@ class BaseDocument(object):
                 value = field.generate()
                 self._data[field_name] = value
 
-            if value is not None:
-                if use_db_field:
-                    data[field.db_field] = value
-                else:
-                    data[field.name] = value
+            if use_db_field:
+                data[field.db_field] = value
+            else:
+                data[field.name] = value
 
         # Only add _cls if allow_inheritance is True
         if not self._meta.get('allow_inheritance'):

From fb213f6e7432e2b9aca4eb4ce619a9db0c521f28 Mon Sep 17 00:00:00 2001
From: Ivan Pogrebkov <chezhik@gmail.com>
Date: Fri, 26 Jan 2018 11:12:02 +0300
Subject: [PATCH 063/121] Update document.py

---
 mongoengine/base/document.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index 956e9b0e..2115a252 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -147,6 +147,7 @@ class BaseDocument(object):
 
             if not hasattr(self, name) and not name.startswith('_'):
                 DynamicField = _import_class('DynamicField')
+                field = DynamicField(db_field=name,null=True)
                 field = DynamicField(db_field=name)
                 field.name = name
                 self._dynamic_fields[name] = field
@@ -337,10 +338,11 @@ class BaseDocument(object):
                 value = field.generate()
                 self._data[field_name] = value
 
-            if use_db_field:
-                data[field.db_field] = value
-            else:
-                data[field.name] = value
+            if (value is not None) or (field.null):
+                if use_db_field:
+                    data[field.db_field] = value
+                else:
+                    data[field.name] = value
 
         # Only add _cls if allow_inheritance is True
         if not self._meta.get('allow_inheritance'):

From 7e8c62104a31bc7168d24e6030437338664340cd Mon Sep 17 00:00:00 2001
From: Ivan Pogrebkov <chezhik@gmail.com>
Date: Fri, 26 Jan 2018 11:15:12 +0300
Subject: [PATCH 064/121] null=True now usefull

---
 mongoengine/document.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index f1622934..d635d62e 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -1010,6 +1010,7 @@ class DynamicDocument(Document):
         field_name = args[0]
         if field_name in self._dynamic_fields:
             setattr(self, field_name, None)
+            self._dynamic_fields[field_name].null = False
         else:
             super(DynamicDocument, self).__delattr__(*args, **kwargs)
 

From fdda27abd1ce957d0bb0e3053027ff0a5ce7b9de Mon Sep 17 00:00:00 2001
From: Esmail <esmail@users.noreply.github.com>
Date: Thu, 1 Feb 2018 12:58:10 -0500
Subject: [PATCH 065/121] Update `post_save` signal documentation to reflect
 #594

---
 docs/guide/signals.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst
index 30277966..eed382c4 100644
--- a/docs/guide/signals.rst
+++ b/docs/guide/signals.rst
@@ -43,10 +43,10 @@ Available signals include:
   has taken place but before saving.
 
 `post_save`
-  Called within :meth:`~mongoengine.Document.save` after all actions
-  (validation, insert/update, cascades, clearing dirty flags) have completed
-  successfully.  Passed the additional boolean keyword argument `created` to
-  indicate if the save was an insert or an update.
+  Called within :meth:`~mongoengine.Document.save` after most actions
+  (validation, insert/update, and cascades, but not clearing dirty flags) have 
+  completed successfully.  Passed the additional boolean keyword argument 
+  `created` to indicate if the save was an insert or an update.
 
 `pre_delete`
   Called within :meth:`~mongoengine.Document.delete` prior to

From 8f6c0796e3c66de67110b9f6cf6ac7e4e92e8915 Mon Sep 17 00:00:00 2001
From: Arto Jantunen <arto@vainu.io>
Date: Fri, 2 Feb 2018 08:20:53 +0200
Subject: [PATCH 066/121] Add db parameter to register_connection

This is done to make it compatible with the connect function.
---
 docs/changelog.rst        | 1 +
 mongoengine/connection.py | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 219d60df..29471463 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -9,6 +9,7 @@ dev
 -  Improve LazyReferenceField and GenericLazyReferenceField with nested fields #1704
 -  Fix validation error instance in GenericEmbeddedDocumentField #1067
 -  Update cached fields when fields argument is given #1712
+-  Add a db parameter to register_connection for compatibility with connect
 
 Changes in 0.15.0
 =================
diff --git a/mongoengine/connection.py b/mongoengine/connection.py
index 34ff4dc3..705dc25b 100644
--- a/mongoengine/connection.py
+++ b/mongoengine/connection.py
@@ -28,7 +28,7 @@ _connections = {}
 _dbs = {}
 
 
-def register_connection(alias, name=None, host=None, port=None,
+def register_connection(alias, db=None, name=None, host=None, port=None,
                         read_preference=READ_PREFERENCE,
                         username=None, password=None,
                         authentication_source=None,
@@ -39,6 +39,7 @@ def register_connection(alias, name=None, host=None, port=None,
     :param alias: the name that will be used to refer to this connection
         throughout MongoEngine
     :param name: the name of the specific database to use
+    :param db: the name of the database to use, for compatibility with connect
     :param host: the host name of the :program:`mongod` instance to connect to
     :param port: the port that the :program:`mongod` instance is running on
     :param read_preference: The read preference for the collection
@@ -58,7 +59,7 @@ def register_connection(alias, name=None, host=None, port=None,
     .. versionchanged:: 0.10.6 - added mongomock support
     """
     conn_settings = {
-        'name': name or 'test',
+        'name': name or db or 'test',
         'host': host or 'localhost',
         'port': port or 27017,
         'read_preference': read_preference,

From 9e80da705a8dae9336be7df02169f36607769fb8 Mon Sep 17 00:00:00 2001
From: Calgary Michael <cseth.michael@gmail.com>
Date: Fri, 2 Feb 2018 21:47:04 -0600
Subject: [PATCH 067/121] removed usage of 'pushAll' operator

---
 mongoengine/queryset/transform.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index 3eadaf64..a9874ddf 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -335,7 +335,7 @@ def update(_doc_cls=None, **update):
                 value = {key: value}
         elif op == 'addToSet' and isinstance(value, list):
             value = {key: {'$each': value}}
-        elif op == 'push':
+        elif op in ('push', 'pushAll'):
             if parts[-1].isdigit():
                 key = parts[0]
                 position = int(parts[-1])
@@ -345,7 +345,13 @@ def update(_doc_cls=None, **update):
                     value = [value]
                 value = {key: {'$each': value, '$position': position}}
             else:
-                value = {key: value}
+                if op == 'pushAll':
+                    op = 'push'  # convert to non-deprecated keyword
+                    if not isinstance(value, (set, tuple, list)):
+                        value = [value]
+                    value = {key: {'$each': value}}
+                else:
+                    value = {key: value}
         else:
             value = {key: value}
         key = '$' + op

From 4d5c6d11ab8665f77aef8ac6e9b2e58843188b75 Mon Sep 17 00:00:00 2001
From: Calgary Michael <cseth.michael@gmail.com>
Date: Fri, 2 Feb 2018 22:04:30 -0600
Subject: [PATCH 068/121] removed deprecated warning for 'update' method

---
 mongoengine/context_managers.py | 11 ++++++++++-
 mongoengine/queryset/base.py    | 17 ++++++++++-------
 tests/queryset/queryset.py      | 11 ++++++-----
 3 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py
index c477575e..e6295570 100644
--- a/mongoengine/context_managers.py
+++ b/mongoengine/context_managers.py
@@ -1,9 +1,11 @@
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
+from pymongo.write_concern import WriteConcern
+from contextlib import contextmanager
 
 
 __all__ = ('switch_db', 'switch_collection', 'no_dereference',
-           'no_sub_classes', 'query_counter')
+           'no_sub_classes', 'query_counter', 'set_write_concern')
 
 
 class switch_db(object):
@@ -215,3 +217,10 @@ class query_counter(object):
         count = self.db.system.profile.find(ignore_query).count() - self.counter
         self.counter += 1
         return count
+
+
+@contextmanager
+def set_write_concern(collection, write_concerns):
+    yield collection.with_options(write_concern=WriteConcern(
+        **dict(collection.write_concern.document.items()),
+        **write_concerns))
diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 6f9c372c..c6e2137c 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -18,7 +18,7 @@ from mongoengine import signals
 from mongoengine.base import get_document
 from mongoengine.common import _import_class
 from mongoengine.connection import get_db
-from mongoengine.context_managers import switch_db
+from mongoengine.context_managers import switch_db, set_write_concern
 from mongoengine.errors import (InvalidQueryError, LookUpError,
                                 NotUniqueError, OperationError)
 from mongoengine.python_support import IS_PYMONGO_3
@@ -510,12 +510,15 @@ class BaseQuerySet(object):
             else:
                 update['$set'] = {'_cls': queryset._document._class_name}
         try:
-            result = queryset._collection.update(query, update, multi=multi,
-                                                 upsert=upsert, **write_concern)
+            with set_write_concern(queryset._collection, write_concern) as collection:
+                update_func = collection.update_one
+                if multi:
+                    update_func = collection.update_many
+                result = update_func(query, update, upsert=upsert)
             if full_result:
                 return result
-            elif result:
-                return result['n']
+            elif result.raw_result:
+                return result.raw_result['n']
         except pymongo.errors.DuplicateKeyError as err:
             raise NotUniqueError(u'Update failed (%s)' % six.text_type(err))
         except pymongo.errors.OperationFailure as err:
@@ -544,10 +547,10 @@ class BaseQuerySet(object):
                                     write_concern=write_concern,
                                     full_result=True, **update)
 
-        if atomic_update['updatedExisting']:
+        if atomic_update.raw_result['updatedExisting']:
             document = self.get()
         else:
-            document = self._document.objects.with_id(atomic_update['upserted'])
+            document = self._document.objects.with_id(atomic_update.upserted_id)
         return document
 
     def update_one(self, upsert=False, write_concern=None, **update):
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 43800fff..848fe35d 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -9,6 +9,7 @@ from nose.plugins.skip import SkipTest
 import pymongo
 from pymongo.errors import ConfigurationError
 from pymongo.read_preferences import ReadPreference
+from pymongo.results import UpdateResult
 import six
 
 from mongoengine import *
@@ -656,14 +657,14 @@ class QuerySetTest(unittest.TestCase):
 
         result = self.Person(name="Bob", age=25).update(
             upsert=True, full_result=True)
-        self.assertTrue(isinstance(result, dict))
-        self.assertTrue("upserted" in result)
-        self.assertFalse(result["updatedExisting"])
+        self.assertTrue(isinstance(result, UpdateResult))
+        self.assertTrue("upserted" in result.raw_result)
+        self.assertFalse(result.raw_result["updatedExisting"])
 
         bob = self.Person.objects.first()
         result = bob.update(set__age=30, full_result=True)
-        self.assertTrue(isinstance(result, dict))
-        self.assertTrue(result["updatedExisting"])
+        self.assertTrue(isinstance(result, UpdateResult))
+        self.assertTrue(result.raw_result["updatedExisting"])
 
         self.Person(name="Bob", age=20).save()
         result = self.Person.objects(name="Bob").update(

From fa38bfd4e8884fe2d6c641fa185c90641c5a24c8 Mon Sep 17 00:00:00 2001
From: Calgary Michael <cseth.michael@gmail.com>
Date: Fri, 2 Feb 2018 22:30:06 -0600
Subject: [PATCH 069/121] made set_write_concern python2.7 compatible

---
 mongoengine/context_managers.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py
index e6295570..cc35cdd7 100644
--- a/mongoengine/context_managers.py
+++ b/mongoengine/context_managers.py
@@ -221,6 +221,6 @@ class query_counter(object):
 
 @contextmanager
 def set_write_concern(collection, write_concerns):
-    yield collection.with_options(write_concern=WriteConcern(
-        **dict(collection.write_concern.document.items()),
-        **write_concerns))
+    old_concerns = dict(collection.write_concern.document.items())
+    combined_concerns = old_concerns.update(write_concerns)
+    yield collection.with_options(write_concern=WriteConcern(**combined_concerns))

From 6835c15d9b0a0379036417e642516f7cdef76840 Mon Sep 17 00:00:00 2001
From: Calgary Michael <cseth.michael@gmail.com>
Date: Fri, 2 Feb 2018 22:41:07 -0600
Subject: [PATCH 070/121] fixing bug in previous commit

---
 mongoengine/context_managers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py
index cc35cdd7..c6d0d40f 100644
--- a/mongoengine/context_managers.py
+++ b/mongoengine/context_managers.py
@@ -221,6 +221,6 @@ class query_counter(object):
 
 @contextmanager
 def set_write_concern(collection, write_concerns):
-    old_concerns = dict(collection.write_concern.document.items())
-    combined_concerns = old_concerns.update(write_concerns)
+    combined_concerns = dict(collection.write_concern.document.items())
+    combined_concerns.update(write_concerns)
     yield collection.with_options(write_concern=WriteConcern(**combined_concerns))

From 38fdf264051430babc8e324f878dc265471b4dea Mon Sep 17 00:00:00 2001
From: Calgary Michael <cseth.michael@gmail.com>
Date: Sun, 4 Feb 2018 10:23:50 -0600
Subject: [PATCH 071/121] added tests for push and push_all

---
 tests/queryset/transform.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py
index 20ab0b3f..20a0c278 100644
--- a/tests/queryset/transform.py
+++ b/tests/queryset/transform.py
@@ -51,6 +51,17 @@ class TransformTest(unittest.TestCase):
         update = transform.update(DicDoc, pull__dictField__test=doc)
         self.assertTrue(isinstance(update["$pull"]["dictField"]["test"], dict))
 
+    def test_transform_update_push(self):
+        """Ensure the differences in behvaior between 'push' and 'push_all'"""
+        class BlogPost(Document):
+            tags = ListField(StringField())
+
+        update = transform.update(BlogPost, push__tags=['mongo', 'db'])
+        self.assertEqual(update, {'$push': {'tags': ['mongo', 'db']}})
+
+        update = transform.update(BlogPost, push_all__tags=['mongo', 'db'])
+        self.assertEqual(update, {'$push': {'tags': {'$each': ['mongo', 'db']}}})
+
     def test_query_field_name(self):
         """Ensure that the correct field name is used when querying.
         """

From 0d854ce906abf87d99ec0a102540570cf800237a Mon Sep 17 00:00:00 2001
From: Ivan Pogrebkov <chezhik@gmail.com>
Date: Mon, 5 Feb 2018 03:24:53 +0300
Subject: [PATCH 072/121] style fix

---
 mongoengine/base/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index 2115a252..172e052f 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -147,7 +147,7 @@ class BaseDocument(object):
 
             if not hasattr(self, name) and not name.startswith('_'):
                 DynamicField = _import_class('DynamicField')
-                field = DynamicField(db_field=name,null=True)
+                field = DynamicField(db_field=name, null=True)
                 field = DynamicField(db_field=name)
                 field.name = name
                 self._dynamic_fields[name] = field

From 6b04ddfad1b2b6574c03ed842e8020feb361d9ce Mon Sep 17 00:00:00 2001
From: Ivan Pogrebkov <chezhik@gmail.com>
Date: Mon, 5 Feb 2018 04:24:03 +0300
Subject: [PATCH 073/121] ><

---
 mongoengine/base/document.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index 172e052f..f31c22ce 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -147,7 +147,6 @@ class BaseDocument(object):
 
             if not hasattr(self, name) and not name.startswith('_'):
                 DynamicField = _import_class('DynamicField')
-                field = DynamicField(db_field=name, null=True)
                 field = DynamicField(db_field=name)
                 field.name = name
                 self._dynamic_fields[name] = field

From de360c61dd239ef44e2d2abd9c4ce835f1144cf8 Mon Sep 17 00:00:00 2001
From: Ivan Pogrebkov <chezhik@gmail.com>
Date: Mon, 5 Feb 2018 04:26:25 +0300
Subject: [PATCH 074/121] removed useless lines

---
 mongoengine/document.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index d635d62e..f1622934 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -1010,7 +1010,6 @@ class DynamicDocument(Document):
         field_name = args[0]
         if field_name in self._dynamic_fields:
             setattr(self, field_name, None)
-            self._dynamic_fields[field_name].null = False
         else:
             super(DynamicDocument, self).__delattr__(*args, **kwargs)
 

From d69808c20414d316fcc6f5a1a6e0d8f751b04a9b Mon Sep 17 00:00:00 2001
From: Ivan Pogrebkov <chezhik@gmail.com>
Date: Mon, 5 Feb 2018 12:33:58 +0300
Subject: [PATCH 075/121] oh, ok...

---
 mongoengine/base/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index f31c22ce..348ee977 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -147,7 +147,7 @@ class BaseDocument(object):
 
             if not hasattr(self, name) and not name.startswith('_'):
                 DynamicField = _import_class('DynamicField')
-                field = DynamicField(db_field=name)
+                field = DynamicField(db_field=name, null=True)
                 field.name = name
                 self._dynamic_fields[name] = field
                 self._fields_ordered += (name,)

From 7efa67e7e6a6bf32d208c6628b3b43a4c0d89bab Mon Sep 17 00:00:00 2001
From: Ivan Pogrebkov <chezhik@gmail.com>
Date: Mon, 5 Feb 2018 12:35:06 +0300
Subject: [PATCH 076/121] reverse to 'style fix'

---
 mongoengine/document.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index f1622934..d635d62e 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -1010,6 +1010,7 @@ class DynamicDocument(Document):
         field_name = args[0]
         if field_name in self._dynamic_fields:
             setattr(self, field_name, None)
+            self._dynamic_fields[field_name].null = False
         else:
             super(DynamicDocument, self).__delattr__(*args, **kwargs)
 

From 7d8916b6e9f60995f905885beac32be321417f1a Mon Sep 17 00:00:00 2001
From: Sangmin In <sm372181@gmail.com>
Date: Sun, 11 Feb 2018 00:59:47 +0900
Subject: [PATCH 077/121] fix case inconsistent casing of "MongoDB" (#1744)

---
 docs/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index ea1a04c1..bcd0d17f 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -86,7 +86,7 @@ of them stand out as particularly intuitive solutions.
 Posts
 ^^^^^
 
-Happily mongoDB *isn't* a relational database, so we're not going to do it that
+Happily MongoDB *isn't* a relational database, so we're not going to do it that
 way. As it turns out, we can use MongoDB's schemaless nature to provide us with
 a much nicer solution. We will store all of the posts in *one collection* and
 each post type will only store the fields it needs. If we later want to add

From 0bd2103a8cb10c63371b2d4cefc8809a5d0adc55 Mon Sep 17 00:00:00 2001
From: Andy Yankovsky <weratt@gmail.com>
Date: Tue, 20 Feb 2018 00:02:12 +0300
Subject: [PATCH 078/121] Add test for document update

---
 tests/document/instance.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index 609bc900..22c44ffa 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -1341,6 +1341,23 @@ class InstanceTest(unittest.TestCase):
         site = Site.objects.first()
         self.assertEqual(site.page.log_message, "Error: Dummy message")
 
+    def test_update_list_field(self):
+        """Test update on `ListField` with $pull + $in.
+        """
+        class Doc(Document):
+            foo = ListField(StringField())
+
+        Doc.drop_collection()
+        doc = Doc(foo=['a', 'b', 'c'])
+        doc.save()
+
+        # Update
+        doc = Doc.objects.first()
+        doc.update(pull__foo__in=['a', 'c'])
+
+        doc = Doc.objects.first()
+        self.assertEqual(doc.foo, ['b'])
+
     def test_embedded_update_db_field(self):
         """Test update on `EmbeddedDocumentField` fields when db_field
         is other than default.

From aa683226416dac3fa4d5fd757091e2c795cf2ff7 Mon Sep 17 00:00:00 2001
From: estein-de <eric_stein@discovery.com>
Date: Tue, 27 Feb 2018 08:43:09 -0600
Subject: [PATCH 079/121] MongoDB wants dates stored in UTC, but the functions
 used in this documentation to generate datetime objects would use server's
 local timezone - fix it! (#1662)

---
 docs/guide/defining-documents.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst
index d41ae7e6..33b5292f 100644
--- a/docs/guide/defining-documents.rst
+++ b/docs/guide/defining-documents.rst
@@ -22,7 +22,7 @@ objects** as class attributes to the document class::
 
     class Page(Document):
         title = StringField(max_length=200, required=True)
-        date_modified = DateTimeField(default=datetime.datetime.now)
+        date_modified = DateTimeField(default=datetime.datetime.utcnow)
 
 As BSON (the binary format for storing data in mongodb) is order dependent,
 documents are serialized based on their field order.
@@ -224,7 +224,7 @@ store; in this situation a :class:`~mongoengine.fields.DictField` is appropriate
         user = ReferenceField(User)
         answers = DictField()
 
-    survey_response = SurveyResponse(date=datetime.now(), user=request.user)
+    survey_response = SurveyResponse(date=datetime.utcnow(), user=request.user)
     response_form = ResponseForm(request.POST)
     survey_response.answers = response_form.cleaned_data()
     survey_response.save()
@@ -618,7 +618,7 @@ collection after a given period. See the official
 documentation for more information.  A common usecase might be session data::
 
     class Session(Document):
-        created = DateTimeField(default=datetime.now)
+        created = DateTimeField(default=datetime.utcnow)
         meta = {
             'indexes': [
                 {'fields': ['created'], 'expireAfterSeconds': 3600}

From a34fd9ac89c3ebd294f57d4b471fb6f5b31e1e91 Mon Sep 17 00:00:00 2001
From: Thomas Erker <35792856+th-erker@users.noreply.github.com>
Date: Thu, 8 Mar 2018 11:25:36 +0000
Subject: [PATCH 080/121] Add testcase for #1751

---
 tests/document/class_methods.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py
index dd3addb7..8701b4b2 100644
--- a/tests/document/class_methods.py
+++ b/tests/document/class_methods.py
@@ -187,6 +187,19 @@ class ClassMethodsTest(unittest.TestCase):
         self.assertEqual(BlogPostWithTags.compare_indexes(), { 'missing': [], 'extra': [] })
         self.assertEqual(BlogPostWithCustomField.compare_indexes(), { 'missing': [], 'extra': [] })
 
+    def test_compare_indexes_for_text_indexes(self):
+        """ Ensure that compare_indexes behaves correctly for text indexes """
+
+        class Doc(Document):
+            a = StringField()
+            meta = { 'indexes': ['$a']}
+
+        Doc.drop_collection()
+        Doc.ensure_indexes()
+        actual = Doc.compare_indexes()
+        expected = {'missing': [], 'extra': []}
+        self.assertEqual(actual, expected)
+
     def test_list_indexes_inheritance(self):
         """ ensure that all of the indexes are listed regardless of the super-
         or sub-class that we call it from

From a0947d0c544a58516650a48b73b84d171e693ffc Mon Sep 17 00:00:00 2001
From: John Dupuy <john@cattailcreek9.com>
Date: Sat, 10 Mar 2018 23:24:04 -0600
Subject: [PATCH 081/121] Edit EmbeddedDocumentListField update() doc

---
 mongoengine/base/datastructures.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py
index 43f32810..fddd945a 100644
--- a/mongoengine/base/datastructures.py
+++ b/mongoengine/base/datastructures.py
@@ -351,7 +351,8 @@ class EmbeddedDocumentList(BaseList):
 
     def update(self, **update):
         """
-        Updates the embedded documents with the given update values.
+        Updates the embedded documents with the given replacement values. This
+        function does not support mongoDB update operators such as ``inc__``.
 
         .. note::
             The embedded document changes are not automatically saved

From dabe8c1bb7e189d0dc8bdceb54eb1ca982a1f6b5 Mon Sep 17 00:00:00 2001
From: Stefan Wojcik <wojcikstefan@gmail.com>
Date: Wed, 14 Mar 2018 14:26:06 -0400
Subject: [PATCH 082/121] highlight places where ValidationError is raised
 outside of validate() method

---
 mongoengine/fields.py | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 7932f73a..f169f0f1 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -614,6 +614,7 @@ class EmbeddedDocumentField(BaseField):
     """
 
     def __init__(self, document_type, **kwargs):
+        # XXX ValidationError raised outside of the "validate" method.
         if not (
             isinstance(document_type, six.string_types) or
             issubclass(document_type, EmbeddedDocument)
@@ -919,8 +920,11 @@ class DictField(ComplexBaseField):
         self.field = field
         self._auto_dereference = False
         self.basecls = basecls or BaseField
+
+        # XXX ValidationError raised outside of the "validate" method.
         if not issubclass(self.basecls, BaseField):
             self.error('DictField only accepts dict values')
+
         kwargs.setdefault('default', lambda: {})
         super(DictField, self).__init__(*args, **kwargs)
 
@@ -969,6 +973,7 @@ class MapField(DictField):
     """
 
     def __init__(self, field=None, *args, **kwargs):
+        # XXX ValidationError raised outside of the "validate" method.
         if not isinstance(field, BaseField):
             self.error('Argument to MapField constructor must be a valid '
                        'field')
@@ -1028,6 +1033,7 @@ class ReferenceField(BaseField):
             A reference to an abstract document type is always stored as a
             :class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`.
         """
+        # XXX ValidationError raised outside of the "validate" method.
         if (
             not isinstance(document_type, six.string_types) and
             not issubclass(document_type, Document)
@@ -1082,6 +1088,8 @@ class ReferenceField(BaseField):
         if isinstance(document, Document):
             # We need the id from the saved object to create the DBRef
             id_ = document.pk
+
+            # XXX ValidationError raised outside of the "validate" method.
             if id_ is None:
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
@@ -1121,7 +1129,6 @@ class ReferenceField(BaseField):
         return self.to_mongo(value)
 
     def validate(self, value):
-
         if not isinstance(value, (self.document_type, LazyReference, DBRef, ObjectId)):
             self.error('A ReferenceField only accepts DBRef, LazyReference, ObjectId or documents')
 
@@ -1129,11 +1136,14 @@ class ReferenceField(BaseField):
             self.error('You can only reference documents once they have been '
                        'saved to the database')
 
-        if self.document_type._meta.get('abstract') and \
-                not isinstance(value, self.document_type):
+        if (
+            self.document_type._meta.get('abstract') and
+            not isinstance(value, self.document_type)
+        ):
             self.error(
                 '%s is not an instance of abstract reference type %s' % (
-                    self.document_type._class_name)
+                    self.document_type._class_name
+                )
             )
 
     def lookup_member(self, member_name):
@@ -1156,6 +1166,7 @@ class CachedReferenceField(BaseField):
         if fields is None:
             fields = []
 
+        # XXX ValidationError raised outside of the "validate" method.
         if (
             not isinstance(document_type, six.string_types) and
             not issubclass(document_type, Document)
@@ -1230,6 +1241,7 @@ class CachedReferenceField(BaseField):
         id_field_name = self.document_type._meta['id_field']
         id_field = self.document_type._fields[id_field_name]
 
+        # XXX ValidationError raised outside of the "validate" method.
         if isinstance(document, Document):
             # We need the id from the saved object to create the DBRef
             id_ = document.pk
@@ -1238,7 +1250,6 @@ class CachedReferenceField(BaseField):
                            ' been saved to the database')
         else:
             self.error('Only accept a document object')
-            # TODO: should raise here or will fail next statement
 
         value = SON((
             ('_id', id_field.to_mongo(id_)),
@@ -1256,6 +1267,7 @@ class CachedReferenceField(BaseField):
         if value is None:
             return None
 
+        # XXX ValidationError raised outside of the "validate" method.
         if isinstance(value, Document):
             if value.pk is None:
                 self.error('You can only reference documents once they have'
@@ -1269,7 +1281,6 @@ class CachedReferenceField(BaseField):
         raise NotImplementedError
 
     def validate(self, value):
-
         if not isinstance(value, self.document_type):
             self.error('A CachedReferenceField only accepts documents')
 
@@ -1330,6 +1341,8 @@ class GenericReferenceField(BaseField):
                 elif isinstance(choice, type) and issubclass(choice, Document):
                     self.choices.append(choice._class_name)
                 else:
+                    # XXX ValidationError raised outside of the "validate"
+                    # method.
                     self.error('Invalid choices provided: must be a list of'
                                'Document subclasses and/or six.string_typess')
 
@@ -1393,6 +1406,7 @@ class GenericReferenceField(BaseField):
             # We need the id from the saved object to create the DBRef
             id_ = document.id
             if id_ is None:
+                # XXX ValidationError raised outside of the "validate" method.
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
         else:
@@ -2209,6 +2223,7 @@ class LazyReferenceField(BaseField):
         automatically call `fetch()` and try to retrive the field on the fetched
         document. Note this only work getting field (not setting or deleting).
         """
+        # XXX ValidationError raised outside of the "validate" method.
         if (
             not isinstance(document_type, six.string_types) and
             not issubclass(document_type, Document)

From e46779f87b8c6cc154d90939f10d35ffd06469ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Malthe=20J=C3=B8rgensen?= <malthe.jorgensen@gmail.com>
Date: Tue, 27 Mar 2018 14:14:08 +0200
Subject: [PATCH 083/121] Docs, queryset.update: `full_result`-arg not clearly
 described

The documentation for the `full_result`-argument to `queryset.update()`
can be read as returning the update documents/objects, whereas it's
really returning just the full "PyMongo result dictionary".

This commit adds some wording and an example dictionary,
to make it clear what the behavior is.
---
 mongoengine/queryset/base.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 6f9c372c..bf8a5b55 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -486,8 +486,9 @@ class BaseQuerySet(object):
             ``save(..., write_concern={w: 2, fsync: True}, ...)`` will
             wait until at least two servers have recorded the write and
             will force an fsync on the primary server.
-        :param full_result: Return the full result rather than just the number
-            updated.
+        :param full_result: Return the full result dictionary rather than just the number
+            updated, e.g. return
+            `{u'n': 2, u'nModified': 2, u'ok': 1.0, 'updatedExisting': True}`.
         :param update: Django-style update keyword arguments
 
         .. versionadded:: 0.2

From 727778b7305cd01c81e3b198fd20079fcb66b0f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Malthe=20J=C3=B8rgensen?= <malthe.jorgensen@gmail.com>
Date: Fri, 30 Mar 2018 20:41:39 +0200
Subject: [PATCH 084/121] Docs, queryset.update(): Fix backtick mistake

Code should be marked with double backticks
---
 mongoengine/queryset/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index bf8a5b55..e5611226 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -488,7 +488,7 @@ class BaseQuerySet(object):
             will force an fsync on the primary server.
         :param full_result: Return the full result dictionary rather than just the number
             updated, e.g. return
-            `{u'n': 2, u'nModified': 2, u'ok': 1.0, 'updatedExisting': True}`.
+            ``{'n': 2, 'nModified': 2, 'ok': 1.0, 'updatedExisting': True}``.
         :param update: Django-style update keyword arguments
 
         .. versionadded:: 0.2

From c6f0d5e4785571980cd8a53c71c27172cb63c7d0 Mon Sep 17 00:00:00 2001
From: Kushal Mitruka <kushal.mit16@gmail.com>
Date: Sun, 1 Apr 2018 20:11:22 +0530
Subject: [PATCH 085/121] fixed pull queries for embeddeddocumentlistfields

Updated mongoengine.queryset.transform.update method to handle EmbeddedDocuementListField during pull operations in DB using mongoegning ORM
fixed : .udpate(pull__emb_doc__emb_doc_list=doc)
---
 mongoengine/queryset/transform.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index 05721850..f96f993c 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -314,11 +314,17 @@ def update(_doc_cls=None, **update):
             field_classes = [c.__class__ for c in cleaned_fields]
             field_classes.reverse()
             ListField = _import_class('ListField')
-            if ListField in field_classes:
-                # Join all fields via dot notation to the last ListField
+            EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
+            if ListField in field_classes or EmbeddedDocumentListField in field_classes:
+                # Join all fields via dot notation to the last ListField or EmbeddedDocumentListField
                 # Then process as normal
+                if ListField in field_classes:
+                    _check_field = ListField
+                else:
+                    _check_field = EmbeddedDocumentListField
+
                 last_listField = len(
-                    cleaned_fields) - field_classes.index(ListField)
+                    cleaned_fields) - field_classes.index(_check_field)
                 key = '.'.join(parts[:last_listField])
                 parts = parts[last_listField:]
                 parts.insert(0, key)

From 806a80cef16ea08b5a33b4ae663b573e7f9a20c4 Mon Sep 17 00:00:00 2001
From: Victor <victor.varvaryuk@mindojo.com>
Date: Thu, 14 Sep 2017 20:31:50 +0300
Subject: [PATCH 086/121] Fixes #1641 There's no need to explicitly raise
 StopIteration as that's what a bare return statement does for a generator
 function - so yes they're the same. As of late 2014 return is correct and
 raise StopIteration for ending a generator is on a depreciation schedule. See
 PEP 479 for full details. https://stackoverflow.com/q/14183803/248296
 https://www.python.org/dev/peps/pep-0479/

---
 mongoengine/queryset/queryset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py
index cf913b01..60de19a2 100644
--- a/mongoengine/queryset/queryset.py
+++ b/mongoengine/queryset/queryset.py
@@ -92,7 +92,7 @@ class QuerySet(BaseQuerySet):
             # Raise StopIteration if we already established there were no more
             # docs in the db cursor.
             if not self._has_more:
-                raise StopIteration
+                return
 
             # Otherwise, populate more of the cache and repeat.
             if len(self._result_cache) <= pos:

From 49bff5d544c30ba0a8433b07cda8f7d4e50c9d73 Mon Sep 17 00:00:00 2001
From: Benjamin Chrobot <benjamin.chrobot@alum.mit.edu>
Date: Thu, 12 Apr 2018 10:47:52 -0400
Subject: [PATCH 087/121] Add documentation for LazyReference and
 GenericLazyReference fields.

---
 docs/apireference.rst             |  2 ++
 docs/guide/defining-documents.rst |  2 ++
 mongoengine/fields.py             | 17 +++++++++++------
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/docs/apireference.rst b/docs/apireference.rst
index 625d4a8b..05ba3f73 100644
--- a/docs/apireference.rst
+++ b/docs/apireference.rst
@@ -87,7 +87,9 @@ Fields
 .. autoclass:: mongoengine.fields.DictField
 .. autoclass:: mongoengine.fields.MapField
 .. autoclass:: mongoengine.fields.ReferenceField
+.. autoclass:: mongoengine.fields.LazyReferenceField
 .. autoclass:: mongoengine.fields.GenericReferenceField
+.. autoclass:: mongoengine.fields.GenericLazyReferenceField
 .. autoclass:: mongoengine.fields.CachedReferenceField
 .. autoclass:: mongoengine.fields.BinaryField
 .. autoclass:: mongoengine.fields.FileField
diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst
index 33b5292f..3ced284e 100644
--- a/docs/guide/defining-documents.rst
+++ b/docs/guide/defining-documents.rst
@@ -80,6 +80,7 @@ are as follows:
 * :class:`~mongoengine.fields.FloatField`
 * :class:`~mongoengine.fields.GenericEmbeddedDocumentField`
 * :class:`~mongoengine.fields.GenericReferenceField`
+* :class:`~mongoengine.fields.GenericLazyReferenceField`
 * :class:`~mongoengine.fields.GeoPointField`
 * :class:`~mongoengine.fields.ImageField`
 * :class:`~mongoengine.fields.IntField`
@@ -87,6 +88,7 @@ are as follows:
 * :class:`~mongoengine.fields.MapField`
 * :class:`~mongoengine.fields.ObjectIdField`
 * :class:`~mongoengine.fields.ReferenceField`
+* :class:`~mongoengine.fields.LazyReferenceField`
 * :class:`~mongoengine.fields.SequenceField`
 * :class:`~mongoengine.fields.SortedListField`
 * :class:`~mongoengine.fields.StringField`
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index f169f0f1..a661874a 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -2204,8 +2204,11 @@ class MultiPolygonField(GeoJsonBaseField):
 
 class LazyReferenceField(BaseField):
     """A really lazy reference to a document.
-    Unlike the :class:`~mongoengine.fields.ReferenceField` it must be manually
-    dereferenced using it ``fetch()`` method.
+    Unlike the :class:`~mongoengine.fields.ReferenceField` it will
+    **not** be automatically (lazily) dereferenced on access.
+    Instead, access will return a :class:`~mongoengine.base.LazyReference` class
+    instance, allowing access to `pk` or manual dereference by using
+    ``fetch()`` method.
 
     .. versionadded:: 0.15
     """
@@ -2331,10 +2334,12 @@ class LazyReferenceField(BaseField):
 
 
 class GenericLazyReferenceField(GenericReferenceField):
-    """A reference to *any* :class:`~mongoengine.document.Document` subclass
-    that will be automatically dereferenced on access (lazily).
-    Unlike the :class:`~mongoengine.fields.GenericReferenceField` it must be
-    manually dereferenced using it ``fetch()`` method.
+    """A reference to *any* :class:`~mongoengine.document.Document` subclass.
+    Unlike the :class:`~mongoengine.fields.GenericReferenceField` it will
+    **not** be automatically (lazily) dereferenced on access.
+    Instead, access will return a :class:`~mongoengine.base.LazyReference` class
+    instance, allowing access to `pk` or manual dereference by using
+    ``fetch()`` method.
 
     .. note ::
         * Any documents used as a generic reference must be registered in the

From faca8512c53102c201cc9b13519ca2ef3c5eb0c5 Mon Sep 17 00:00:00 2001
From: Emmanuel Nosa Evbuomwan <emmanuelnosa.e@gmail.com>
Date: Sun, 29 Apr 2018 17:32:03 +0100
Subject: [PATCH 088/121] Updated text-indexes.rst

The search statement under the `Ordering by text score` section uses `search` on the QuerySet object instead of `search_text` and thus raised an `AttributeError`

AttributeError: 'QuerySet' object has no attribute 'search'
---
 docs/guide/text-indexes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/guide/text-indexes.rst b/docs/guide/text-indexes.rst
index 725ad369..92a4471a 100644
--- a/docs/guide/text-indexes.rst
+++ b/docs/guide/text-indexes.rst
@@ -48,4 +48,4 @@ Ordering by text score
 
 ::
 
-  objects = News.objects.search('mongo').order_by('$text_score')
+  objects = News.objects.search_text('mongo').order_by('$text_score')

From 65e4fea4efbb2f888bcf15850777457aafbd5720 Mon Sep 17 00:00:00 2001
From: Kushal Mitruka <kushal.mit16@gmail.com>
Date: Tue, 1 May 2018 20:32:38 +0530
Subject: [PATCH 089/121] added test cases for update pull queries

---
 tests/queryset/transform.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py
index a043a647..a2636122 100644
--- a/tests/queryset/transform.py
+++ b/tests/queryset/transform.py
@@ -247,7 +247,31 @@ class TransformTest(unittest.TestCase):
         events = Event.objects(location__within=box)
         with self.assertRaises(InvalidQueryError):
             events.count()
+    
+    def test_update_pull_for_list_fields(self):
+        """ 
+        Test added to check pull operation in update for 
+        EmbeddedDocumentListField which is inside a EmbeddedDocumentField
+        """
+        class Word(EmbeddedDocument):
+            word = StringField()
+            index = IntField()
+        
+        class SubDoc(EmbeddedDocument):
+            heading = ListField(StringField())
+            text = EmbeddedDocumentListField(Word)
+        
+        class MainDoc(Document):
+            title = StringField()
+            content = EmbeddedDocumentField(SubDoc)
+        
+        word = Word(word='abc', index=1)
+        update = transform.update(MainDoc, pull__content__text=word)
+        self.assertEqual(update, {'$pull': {'content.text': SON([('word', u'abc'), ('index', 1)])}})
 
-
+        update = transform.update(MainDoc, pull__content__heading='xyz')
+        self.assertEqual(update, {'$pull': {'content.heading': 'xyz'}})
+        
+        
 if __name__ == '__main__':
     unittest.main()

From b1f62a27353a1c2bd8d69b4697dab16caa8056cf Mon Sep 17 00:00:00 2001
From: Kushal Mitruka <kushal.mit16@gmail.com>
Date: Tue, 1 May 2018 21:08:43 +0530
Subject: [PATCH 090/121] added import in tests/queryset/transform.py for SON
 object

---
 tests/queryset/transform.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py
index a2636122..a1f060d5 100644
--- a/tests/queryset/transform.py
+++ b/tests/queryset/transform.py
@@ -1,5 +1,7 @@
 import unittest
 
+from bson.son import SON
+
 from mongoengine import *
 from mongoengine.queryset import Q, transform
 

From 5a6d4387ea7cbc6eea6176d7c3d67cdcc64c2b31 Mon Sep 17 00:00:00 2001
From: Andy Yankovsky <weratt@gmail.com>
Date: Mon, 7 May 2018 23:17:12 +0300
Subject: [PATCH 091/121] Restore comment from cached value after cursor copy

---
 mongoengine/queryset/base.py |  3 +++
 tests/queryset/queryset.py   | 15 ++++++++++-----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index e5611226..b1d3342b 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -1579,6 +1579,9 @@ class BaseQuerySet(object):
         if self._batch_size is not None:
             self._cursor_obj.batch_size(self._batch_size)
 
+        if self._comment is not None:
+            self._cursor_obj.comment(self._comment)
+
         return self._cursor_obj
 
     def __deepcopy__(self, memo):
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 43800fff..c9b87eeb 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -2383,14 +2383,19 @@ class QuerySetTest(unittest.TestCase):
             age = IntField()
 
         with db_ops_tracker() as q:
-            adult = (User.objects.filter(age__gte=18)
+            adult1 = (User.objects.filter(age__gte=18)
                 .comment('looking for an adult')
                 .first())
+
+            adult2 = (User.objects.comment('looking for an adult')
+                .filter(age__gte=18)
+                .first())
+
             ops = q.get_ops()
-            self.assertEqual(len(ops), 1)
-            op = ops[0]
-            self.assertEqual(op['query']['$query'], {'age': {'$gte': 18}})
-            self.assertEqual(op['query']['$comment'], 'looking for an adult')
+            self.assertEqual(len(ops), 2)
+            for op in ops:
+                self.assertEqual(op['query']['$query'], {'age': {'$gte': 18}})
+                self.assertEqual(op['query']['$comment'], 'looking for an adult')
 
     def test_map_reduce(self):
         """Ensure map/reduce is both mapping and reducing.

From 85d621846d521e2662d420983393b49737a22ae3 Mon Sep 17 00:00:00 2001
From: Benjamin Jiang <jbjsnow@outlook.com>
Date: Wed, 16 May 2018 11:37:32 +0800
Subject: [PATCH 092/121] Fix typo

---
 mongoengine/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index c948dac2..a8061749 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -579,7 +579,7 @@ class Document(BaseDocument):
         """Delete the :class:`~mongoengine.Document` from the database. This
         will only take effect if the document has been previously saved.
 
-        :parm signal_kwargs: (optional) kwargs dictionary to be passed to
+        :param signal_kwargs: (optional) kwargs dictionary to be passed to
             the signal calls.
         :param write_concern: Extra keyword arguments are passed down which
             will be used as options for the resultant

From 08b64338439a33a1b869ef0c77f09fd9b50bae2c Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 11:03:13 +0800
Subject: [PATCH 093/121] fix compare_indexes for text indexes #1751

---
 mongoengine/document.py         | 12 ++++++++++--
 tests/document/class_methods.py |  8 +++++++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index c948dac2..18c52d9d 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -967,8 +967,16 @@ class Document(BaseDocument):
         """
 
         required = cls.list_indexes()
-        existing = [info['key']
-                    for info in cls._get_collection().index_information().values()]
+
+        existing = []
+        for info in cls._get_collection().index_information().values():
+            if '_fts' in info['key'][0]:
+                index_type = info['key'][0][1]
+                text_index_fields = info.get('weights').keys()
+                existing.append(
+                    [(key, index_type) for key in text_index_fields])
+            else:
+                existing.append(info['key'])
         missing = [index for index in required if index not in existing]
         extra = [index for index in existing if index not in required]
 
diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py
index 8701b4b2..3f052d45 100644
--- a/tests/document/class_methods.py
+++ b/tests/document/class_methods.py
@@ -192,7 +192,13 @@ class ClassMethodsTest(unittest.TestCase):
 
         class Doc(Document):
             a = StringField()
-            meta = { 'indexes': ['$a']}
+            b = StringField()
+            meta = {'indexes': [
+                {'fields': ['$a', "$b"],
+                 'default_language': 'english',
+                 'weights': {'a': 10, 'b': 2}
+                }
+            ]}
 
         Doc.drop_collection()
         Doc.ensure_indexes()

From e50d66b3032df29331f72f6ce238b3813ae35b50 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 11:26:30 +0800
Subject: [PATCH 094/121] skip mongodb 2.4

---
 tests/document/class_methods.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py
index 3f052d45..2fab1f72 100644
--- a/tests/document/class_methods.py
+++ b/tests/document/class_methods.py
@@ -5,6 +5,7 @@ from mongoengine import *
 
 from mongoengine.queryset import NULLIFY, PULL
 from mongoengine.connection import get_db
+from tests.utils import needs_mongodb_v26
 
 __all__ = ("ClassMethodsTest", )
 
@@ -187,6 +188,7 @@ class ClassMethodsTest(unittest.TestCase):
         self.assertEqual(BlogPostWithTags.compare_indexes(), { 'missing': [], 'extra': [] })
         self.assertEqual(BlogPostWithCustomField.compare_indexes(), { 'missing': [], 'extra': [] })
 
+    @needs_mongodb_v26
     def test_compare_indexes_for_text_indexes(self):
         """ Ensure that compare_indexes behaves correctly for text indexes """
 

From 257a43298b93785d9280d3942745aab7149bf627 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 12:31:27 +0800
Subject: [PATCH 095/121] use MongoClient.is_mongos in ensure indexes #1759

---
 mongoengine/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index 2de0b1a3..865bb063 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -854,7 +854,7 @@ class Document(BaseDocument):
         collection = cls._get_collection()
         # 746: when connection is via mongos, the read preference is not necessarily an indication that
         # this code runs on a secondary
-        if not collection.is_mongos and collection.read_preference > 1:
+        if not collection.database.client.is_mongos and collection.read_preference > 1:
             return
 
         # determine if an index which we are creating includes

From fd02d77c59aa6c427581c9903d158c1d77cbfa26 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 12:43:24 +0800
Subject: [PATCH 096/121] drop pymongo 2.x support in update

---
 .travis.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a70c711e..29a72d1c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,8 +19,6 @@ python:
 - pypy
 
 env:
-- MONGODB=2.6 PYMONGO=2.7
-- MONGODB=2.6 PYMONGO=2.8
 - MONGODB=2.6 PYMONGO=3.x
 
 matrix:
@@ -28,14 +26,10 @@ matrix:
   fast_finish: true
 
   include:
-  - python: 2.7
-    env: MONGODB=2.4 PYMONGO=2.7
   - python: 2.7
     env: MONGODB=2.4 PYMONGO=3.5
   - python: 2.7
     env: MONGODB=3.0 PYMONGO=3.x
-  - python: 3.5
-    env: MONGODB=2.4 PYMONGO=2.7
   - python: 3.5
     env: MONGODB=2.4 PYMONGO=3.5
   - python: 3.5

From f605eb14e8b9e4c02d8279a54035f1c00791cf22 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 12:54:24 +0800
Subject: [PATCH 097/121] fix style

---
 mongoengine/context_managers.py | 4 ++--
 mongoengine/document.py         | 2 +-
 tox.ini                         | 4 +---
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py
index c6d0d40f..150f9657 100644
--- a/mongoengine/context_managers.py
+++ b/mongoengine/context_managers.py
@@ -1,7 +1,7 @@
 from mongoengine.common import _import_class
-from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
-from pymongo.write_concern import WriteConcern
 from contextlib import contextmanager
+from pymongo.write_concern import WriteConcern
+from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 
 
 __all__ = ('switch_db', 'switch_collection', 'no_dereference',
diff --git a/mongoengine/document.py b/mongoengine/document.py
index 865bb063..2de0b1a3 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -854,7 +854,7 @@ class Document(BaseDocument):
         collection = cls._get_collection()
         # 746: when connection is via mongos, the read preference is not necessarily an indication that
         # this code runs on a secondary
-        if not collection.database.client.is_mongos and collection.read_preference > 1:
+        if not collection.is_mongos and collection.read_preference > 1:
             return
 
         # determine if an index which we are creating includes
diff --git a/tox.ini b/tox.ini
index 9bb0c5ec..2f2b1757 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,13 +1,11 @@
 [tox]
-envlist = {py27,py35,pypy,pypy3}-{mg27,mg28,mg35,mg3x}
+envlist = {py27,py35,pypy,pypy3}-{mg35,mg3x}
 
 [testenv]
 commands =
     python setup.py nosetests {posargs}
 deps =
     nose
-    mg27: PyMongo<2.8
-    mg28: PyMongo>=2.8,<2.9
     mg35: PyMongo==3.5
     mg3x: PyMongo>=3.0
 setenv =

From c83c6350676af155a8c204a0593e74ee8b674e6b Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 13:04:51 +0800
Subject: [PATCH 098/121] fix import order

---
 mongoengine/context_managers.py | 2 +-
 mongoengine/queryset/base.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py
index 150f9657..ec2e9e8b 100644
--- a/mongoengine/context_managers.py
+++ b/mongoengine/context_managers.py
@@ -1,6 +1,6 @@
-from mongoengine.common import _import_class
 from contextlib import contextmanager
 from pymongo.write_concern import WriteConcern
+from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 
 
diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 2a1bf3ad..f78ee882 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -18,7 +18,7 @@ from mongoengine import signals
 from mongoengine.base import get_document
 from mongoengine.common import _import_class
 from mongoengine.connection import get_db
-from mongoengine.context_managers import switch_db, set_write_concern
+from mongoengine.context_managers import set_write_concern, switch_db
 from mongoengine.errors import (InvalidQueryError, LookUpError,
                                 NotUniqueError, OperationError)
 from mongoengine.python_support import IS_PYMONGO_3

From 843fc03bf490b6c6346a22f8599046f718631beb Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 13:16:25 +0800
Subject: [PATCH 099/121] add changelog for update_one,update_many

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 29471463..cb31da21 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -10,6 +10,7 @@ dev
 -  Fix validation error instance in GenericEmbeddedDocumentField #1067
 -  Update cached fields when fields argument is given #1712
 -  Add a db parameter to register_connection for compatibility with connect
+-  Use new update_one, update_many on document/queryset update #1491
 
 Changes in 0.15.0
 =================

From 3e0d84383e4a0f9c52d6840ff1d4a882e7eeb6cb Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 13:41:20 +0800
Subject: [PATCH 100/121] use insert_one, insert_many and remove deprecated one
 #1491

---
 docs/changelog.rst           | 1 +
 mongoengine/queryset/base.py | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 29471463..4dec42ce 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -10,6 +10,7 @@ dev
 -  Fix validation error instance in GenericEmbeddedDocumentField #1067
 -  Update cached fields when fields argument is given #1712
 -  Add a db parameter to register_connection for compatibility with connect
+-  Use insert_one, insert_many in Document.insert #1491
 
 Changes in 0.15.0
 =================
diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index b1d3342b..95b25d06 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -350,8 +350,14 @@ class BaseQuerySet(object):
                                      documents=docs, **signal_kwargs)
 
         raw = [doc.to_mongo() for doc in docs]
+        insert_func = self._collection.insert_many
+        if return_one:
+            raw = raw[0]
+            insert_func = self._collection.insert_one
+
         try:
-            ids = self._collection.insert(raw, **write_concern)
+            inserted_result = insert_func(raw, **write_concern)
+            ids = inserted_result.inserted_id if return_one else inserted_result.inserted_ids
         except pymongo.errors.DuplicateKeyError as err:
             message = 'Could not save document (%s)'
             raise NotUniqueError(message % six.text_type(err))

From f9a887c8c6665a77dda33ca785acf2ece8158337 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 14:33:12 +0800
Subject: [PATCH 101/121] fix inserted_ids

---
 mongoengine/queryset/base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 95b25d06..eec7df18 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -357,7 +357,8 @@ class BaseQuerySet(object):
 
         try:
             inserted_result = insert_func(raw, **write_concern)
-            ids = inserted_result.inserted_id if return_one else inserted_result.inserted_ids
+            ids = return_one and inserted_result.inserted_id or\
+                  inserted_result.inserted_ids
         except pymongo.errors.DuplicateKeyError as err:
             message = 'Could not save document (%s)'
             raise NotUniqueError(message % six.text_type(err))
@@ -374,7 +375,6 @@ class BaseQuerySet(object):
             signals.post_bulk_insert.send(
                 self._document, documents=docs, loaded=False, **signal_kwargs)
             return return_one and ids[0] or ids
-
         documents = self.in_bulk(ids)
         results = []
         for obj_id in ids:

From 1d3f20b666bfe0ef8dcdea06a2341e460d13c486 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 14:41:25 +0800
Subject: [PATCH 102/121] fix style and ids need to be an array

---
 mongoengine/queryset/base.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 2f2d38dd..0634dceb 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -357,8 +357,7 @@ class BaseQuerySet(object):
 
         try:
             inserted_result = insert_func(raw, **write_concern)
-            ids = return_one and inserted_result.inserted_id or\
-                  inserted_result.inserted_ids
+            ids = return_one and [inserted_result.inserted_id] or inserted_result.inserted_ids
         except pymongo.errors.DuplicateKeyError as err:
             message = 'Could not save document (%s)'
             raise NotUniqueError(message % six.text_type(err))

From 1aebc9514564a9ba1c58cb52fa4552ace6d2066a Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 13:41:20 +0800
Subject: [PATCH 103/121] use insert_one, insert_many and remove deprecated one
 #1491

---
 docs/changelog.rst           | 1 +
 mongoengine/queryset/base.py | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index cb31da21..e3d366b3 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -11,6 +11,7 @@ dev
 -  Update cached fields when fields argument is given #1712
 -  Add a db parameter to register_connection for compatibility with connect
 -  Use new update_one, update_many on document/queryset update #1491
+-  Use insert_one, insert_many in Document.insert #1491
 
 Changes in 0.15.0
 =================
diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index f78ee882..ff7afa96 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -350,8 +350,14 @@ class BaseQuerySet(object):
                                      documents=docs, **signal_kwargs)
 
         raw = [doc.to_mongo() for doc in docs]
+        insert_func = self._collection.insert_many
+        if return_one:
+            raw = raw[0]
+            insert_func = self._collection.insert_one
+
         try:
-            ids = self._collection.insert(raw, **write_concern)
+            inserted_result = insert_func(raw, **write_concern)
+            ids = inserted_result.inserted_id if return_one else inserted_result.inserted_ids
         except pymongo.errors.DuplicateKeyError as err:
             message = 'Could not save document (%s)'
             raise NotUniqueError(message % six.text_type(err))

From 5c0bd8a81090aa39a74b347adb7afde9edb202b6 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 14:33:12 +0800
Subject: [PATCH 104/121] fix inserted_ids

---
 mongoengine/queryset/base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index ff7afa96..2f2d38dd 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -357,7 +357,8 @@ class BaseQuerySet(object):
 
         try:
             inserted_result = insert_func(raw, **write_concern)
-            ids = inserted_result.inserted_id if return_one else inserted_result.inserted_ids
+            ids = return_one and inserted_result.inserted_id or\
+                  inserted_result.inserted_ids
         except pymongo.errors.DuplicateKeyError as err:
             message = 'Could not save document (%s)'
             raise NotUniqueError(message % six.text_type(err))
@@ -374,7 +375,6 @@ class BaseQuerySet(object):
             signals.post_bulk_insert.send(
                 self._document, documents=docs, loaded=False, **signal_kwargs)
             return return_one and ids[0] or ids
-
         documents = self.in_bulk(ids)
         results = []
         for obj_id in ids:

From 0fc55451c2aaad6a612d28ead0b58898a2236e40 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 14:41:25 +0800
Subject: [PATCH 105/121] fix style and ids need to be an array

---
 mongoengine/queryset/base.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 2f2d38dd..0634dceb 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -357,8 +357,7 @@ class BaseQuerySet(object):
 
         try:
             inserted_result = insert_func(raw, **write_concern)
-            ids = return_one and inserted_result.inserted_id or\
-                  inserted_result.inserted_ids
+            ids = return_one and [inserted_result.inserted_id] or inserted_result.inserted_ids
         except pymongo.errors.DuplicateKeyError as err:
             message = 'Could not save document (%s)'
             raise NotUniqueError(message % six.text_type(err))

From acba86993dd92b04fff289a151cb8882913bbf20 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 15:43:19 +0800
Subject: [PATCH 106/121] set_write_concern pymongo3

---
 mongoengine/queryset/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 0634dceb..0a3f65fb 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -356,7 +356,7 @@ class BaseQuerySet(object):
             insert_func = self._collection.insert_one
 
         try:
-            inserted_result = insert_func(raw, **write_concern)
+            inserted_result = insert_func(raw, set_write_concern(write_concern))
             ids = return_one and [inserted_result.inserted_id] or inserted_result.inserted_ids
         except pymongo.errors.DuplicateKeyError as err:
             message = 'Could not save document (%s)'

From fa4ac95ecc245c59040664905c935d3c85b49429 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 17:49:49 +0800
Subject: [PATCH 107/121] catch bulkwriteerror

---
 mongoengine/queryset/base.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 008e7f15..391cc819 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -371,6 +371,11 @@ class BaseQuerySet(object):
                 message = u'Tried to save duplicate unique keys (%s)'
                 raise NotUniqueError(message % six.text_type(err))
             raise OperationError(message % six.text_type(err))
+        except pymongo.error.BulkWriteError as err:
+            # inserting documents that already have an _id field will
+            # give huge performance debt or raise
+            message = u'Document must not have _id value before bulk write (%s)'
+            raise NotUniqueError(message % sx.text_type(err))
 
         if not load_bulk:
             signals.post_bulk_insert.send(

From 78601d90c9fd13b20d1e6cfe2342ddb23a520e33 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 17:54:13 +0800
Subject: [PATCH 108/121] fix typo

---
 mongoengine/queryset/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 391cc819..4c8309a7 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -375,7 +375,7 @@ class BaseQuerySet(object):
             # inserting documents that already have an _id field will
             # give huge performance debt or raise
             message = u'Document must not have _id value before bulk write (%s)'
-            raise NotUniqueError(message % sx.text_type(err))
+            raise NotUniqueError(message % six.text_type(err))
 
         if not load_bulk:
             signals.post_bulk_insert.send(

From 94cda90a6ead7a64059328c58dafa6154436c951 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 17:56:19 +0800
Subject: [PATCH 109/121] fix syntax

---
 mongoengine/queryset/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index 4c8309a7..bba63da0 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -371,7 +371,7 @@ class BaseQuerySet(object):
                 message = u'Tried to save duplicate unique keys (%s)'
                 raise NotUniqueError(message % six.text_type(err))
             raise OperationError(message % six.text_type(err))
-        except pymongo.error.BulkWriteError as err:
+        except pymongo.errors.BulkWriteError as err:
             # inserting documents that already have an _id field will
             # give huge performance debt or raise
             message = u'Document must not have _id value before bulk write (%s)'

From 088fd6334bfe8316c2df2466ad93ee568e3013c6 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 18:06:59 +0800
Subject: [PATCH 110/121] bulkwriteerror does not trigger

---
 mongoengine/queryset/base.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index bba63da0..647eafc6 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -363,6 +363,11 @@ class BaseQuerySet(object):
         except pymongo.errors.DuplicateKeyError as err:
             message = 'Could not save document (%s)'
             raise NotUniqueError(message % six.text_type(err))
+        except pymongo.errors.BulkWriteError as err:
+            # inserting documents that already have an _id field will
+            # give huge performance debt or raise
+            message = u'Document must not have _id value before bulk write (%s)'
+            raise NotUniqueError(message % six.text_type(err))
         except pymongo.errors.OperationFailure as err:
             message = 'Could not save document (%s)'
             if re.match('^E1100[01] duplicate key', six.text_type(err)):
@@ -371,11 +376,6 @@ class BaseQuerySet(object):
                 message = u'Tried to save duplicate unique keys (%s)'
                 raise NotUniqueError(message % six.text_type(err))
             raise OperationError(message % six.text_type(err))
-        except pymongo.errors.BulkWriteError as err:
-            # inserting documents that already have an _id field will
-            # give huge performance debt or raise
-            message = u'Document must not have _id value before bulk write (%s)'
-            raise NotUniqueError(message % six.text_type(err))
 
         if not load_bulk:
             signals.post_bulk_insert.send(

From 506168ab83a11dc22c706dd308f77f6233543d57 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 18:25:50 +0800
Subject: [PATCH 111/121] use write_concern class

---
 tests/queryset/queryset.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 5c3f179f..4c0085d0 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -927,8 +927,7 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertEqual(Blog.objects.count(), 2)
 
-        Blog.objects.insert([blog2, blog3],
-                            write_concern={"w": 0, 'continue_on_error': True})
+        Blog.objects.insert([blog2, blog3], write_concern={"w": 0})
         self.assertEqual(Blog.objects.count(), 3)
 
     def test_get_changed_fields_query_count(self):

From fc5d9ae10033e7e083931d9a64b7137feadc6eb6 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Sun, 20 May 2018 18:39:22 +0800
Subject: [PATCH 112/121] pymongo3 does not support continue_on_error

---
 tests/queryset/queryset.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 4c0085d0..3fa86a58 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -927,9 +927,6 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertEqual(Blog.objects.count(), 2)
 
-        Blog.objects.insert([blog2, blog3], write_concern={"w": 0})
-        self.assertEqual(Blog.objects.count(), 3)
-
     def test_get_changed_fields_query_count(self):
         """Make sure we don't perform unnecessary db operations when
         none of document's fields were updated.

From 2adb640821123439c9b611375d01aba1d86b3b26 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Mon, 21 May 2018 09:19:03 +0800
Subject: [PATCH 113/121] modify bulk_insert test for pymongo3

---
 tests/queryset/queryset.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 3fa86a58..50825e5c 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -846,10 +846,10 @@ class QuerySetTest(unittest.TestCase):
 
             Blog.objects.insert(blogs, load_bulk=False)
             if mongodb_version < (2, 6):
-                self.assertEqual(q, 1)
+                self.assertEqual(q, 99)
             else:
                 # profiling logs each doc now in the bulk op
-                self.assertEqual(q, 99)
+                self.assertEqual(q, 1)
 
         Blog.drop_collection()
         Blog.ensure_indexes()
@@ -859,7 +859,7 @@ class QuerySetTest(unittest.TestCase):
 
             Blog.objects.insert(blogs)
             if mongodb_version < (2, 6):
-                self.assertEqual(q, 2)  # 1 for insert, and 1 for in bulk fetch
+                self.assertEqual(q, 101)  # 100 for insert, and 1 for in bulk fetch
             else:
                 # 99 for insert, and 1 for in bulk fetch
                 self.assertEqual(q, 100)

From e4451ccaf8616e27c3479e97bcb75daf76396488 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Mon, 21 May 2018 09:22:33 +0800
Subject: [PATCH 114/121] insert_many uses only one insert

---
 tests/queryset/queryset.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 50825e5c..8e91feae 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -845,11 +845,8 @@ class QuerySetTest(unittest.TestCase):
                 blogs.append(Blog(title="post %s" % i, posts=[post1, post2]))
 
             Blog.objects.insert(blogs, load_bulk=False)
-            if mongodb_version < (2, 6):
-                self.assertEqual(q, 99)
-            else:
-                # profiling logs each doc now in the bulk op
-                self.assertEqual(q, 1)
+            # profiling logs each doc now in the bulk op
+            self.assertEqual(q, 99)
 
         Blog.drop_collection()
         Blog.ensure_indexes()
@@ -861,8 +858,8 @@ class QuerySetTest(unittest.TestCase):
             if mongodb_version < (2, 6):
                 self.assertEqual(q, 101)  # 100 for insert, and 1 for in bulk fetch
             else:
-                # 99 for insert, and 1 for in bulk fetch
-                self.assertEqual(q, 100)
+                # 1 for insert, and 1 for in bulk fetch
+                self.assertEqual(q, 2)
 
         Blog.drop_collection()
 

From f7a3acfaf45673ee379fc09de86f4fa484e907bc Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Mon, 21 May 2018 09:34:44 +0800
Subject: [PATCH 115/121] query profiler test fix

---
 tests/queryset/queryset.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 8e91feae..9c7ac0e4 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -855,11 +855,7 @@ class QuerySetTest(unittest.TestCase):
             self.assertEqual(q, 0)
 
             Blog.objects.insert(blogs)
-            if mongodb_version < (2, 6):
-                self.assertEqual(q, 101)  # 100 for insert, and 1 for in bulk fetch
-            else:
-                # 1 for insert, and 1 for in bulk fetch
-                self.assertEqual(q, 2)
+            self.assertEqual(q, 100) # 99 for insert 1 for fetch
 
         Blog.drop_collection()
 

From 3f9ff7254f206e0926c6ab31cc3a1a1efe39ad2d Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Mon, 21 May 2018 09:46:23 +0800
Subject: [PATCH 116/121] fix queryset tests

---
 tests/queryset/queryset.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 9c7ac0e4..dea5b110 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -970,7 +970,10 @@ class QuerySetTest(unittest.TestCase):
         org = Organization.objects.get(id=o1.id)
         with query_counter() as q:
             org.save(cascade=False)
-            self.assertEqual(q, 0)
+            if mongodb_version >= (3, 0):
+                self.assertEqual(q, 1)
+            else:
+                self.assertEqual(q, 0)
 
         # Saving a doc after you append a reference to it should result in
         # two db operations (a query for the reference and an update).

From 6fb5c312c3a8d753c4a49a6e65bbc9db3d84f5bb Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Mon, 21 May 2018 09:54:19 +0800
Subject: [PATCH 117/121] fix test error

---
 tests/queryset/queryset.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index dea5b110..2b12d261 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -967,6 +967,11 @@ class QuerySetTest(unittest.TestCase):
             org.save()
             self.assertEqual(q, 0)
 
+        # get MongoDB version info
+        connection = get_connection()
+        info = connection.test.command('buildInfo')
+        mongodb_version = tuple([int(i) for i in info['version'].split('.')])
+
         org = Organization.objects.get(id=o1.id)
         with query_counter() as q:
             org.save(cascade=False)

From 9bd328e147122133a64f00a71c5829c8803217ed Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Mon, 21 May 2018 10:04:59 +0800
Subject: [PATCH 118/121] query_counter fix

---
 tests/queryset/queryset.py | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
index 2b12d261..9c7ac0e4 100644
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -967,18 +967,10 @@ class QuerySetTest(unittest.TestCase):
             org.save()
             self.assertEqual(q, 0)
 
-        # get MongoDB version info
-        connection = get_connection()
-        info = connection.test.command('buildInfo')
-        mongodb_version = tuple([int(i) for i in info['version'].split('.')])
-
         org = Organization.objects.get(id=o1.id)
         with query_counter() as q:
             org.save(cascade=False)
-            if mongodb_version >= (3, 0):
-                self.assertEqual(q, 1)
-            else:
-                self.assertEqual(q, 0)
+            self.assertEqual(q, 0)
 
         # Saving a doc after you append a reference to it should result in
         # two db operations (a query for the reference and an update).

From 290b821a3ae2e130192a90bb62ad5ae8d5461005 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Sat, 2 Sep 2017 02:05:27 +0900
Subject: [PATCH 119/121] add fix for reload(fields) affect changed fields
 #1371

---
 mongoengine/document.py    |  5 +++--
 tests/document/instance.py | 20 +++++++++++++++++++-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index 2de0b1a3..182733c7 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -705,7 +705,6 @@ class Document(BaseDocument):
             obj = obj[0]
         else:
             raise self.DoesNotExist('Document does not exist')
-
         for field in obj._data:
             if not fields or field in fields:
                 try:
@@ -721,7 +720,9 @@ class Document(BaseDocument):
                         # i.e. obj.update(unset__field=1) followed by obj.reload()
                         delattr(self, field)
 
-        self._changed_fields = obj._changed_fields
+        self._changed_fields = list(
+            set(self._changed_fields) - set(fields)
+            ) if fields else obj._changed_fields
         self._created = False
         return self
 
diff --git a/tests/document/instance.py b/tests/document/instance.py
index 38c7fcaf..b255e8a6 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -476,6 +476,24 @@ class InstanceTest(unittest.TestCase):
         doc.save()
         doc.reload()
 
+    def test_reload_with_changed_fields(self):
+        """Ensures reloading will not affect changed fields"""
+        class User(Document):
+            name = StringField()
+            number = IntField()
+        User.drop_collection()
+
+        user = User(name="Bob", number=1).save()
+        user.name = "John"
+        user.number = 2
+
+        self.assertEqual(user._get_changed_fields(), ['name', 'number'])
+        user.reload('number')
+        self.assertEqual(user._get_changed_fields(), ['name'])
+        user.save()
+        user.reload()
+        self.assertEqual(user.name, "John")
+
     def test_reload_referencing(self):
         """Ensures reloading updates weakrefs correctly."""
         class Embedded(EmbeddedDocument):
@@ -521,7 +539,7 @@ class InstanceTest(unittest.TestCase):
         doc.save()
         doc.dict_field['extra'] = 1
         doc = doc.reload(10, 'list_field')
-        self.assertEqual(doc._get_changed_fields(), [])
+        self.assertEqual(doc._get_changed_fields(), ['dict_field.extra'])
         self.assertEqual(len(doc.list_field), 5)
         self.assertEqual(len(doc.dict_field), 3)
         self.assertEqual(len(doc.embedded_field.list_field), 4)

From d424583cbf26de5c9b4f3ac47ec9b4cb52549fd5 Mon Sep 17 00:00:00 2001
From: Erdenezul <erdenezul@shine.mn>
Date: Sat, 2 Sep 2017 12:00:57 +0900
Subject: [PATCH 120/121] fix flake8 error #1371

---
 mongoengine/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/document.py b/mongoengine/document.py
index 182733c7..109f8f82 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -722,7 +722,7 @@ class Document(BaseDocument):
 
         self._changed_fields = list(
             set(self._changed_fields) - set(fields)
-            ) if fields else obj._changed_fields
+        ) if fields else obj._changed_fields
         self._created = False
         return self
 

From 784386fddcc3d171791fb710e332d72d29c009a5 Mon Sep 17 00:00:00 2001
From: Erdenezul Batmunkh <erdenezul.b@gmail.com>
Date: Mon, 21 May 2018 16:07:08 +0800
Subject: [PATCH 121/121] add changelog #1371

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index d8765d17..08e5a490 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -13,6 +13,7 @@ dev
 -  Use insert_one, insert_many in Document.insert #1491
 -  Use new update_one, update_many on document/queryset update #1491
 -  Use insert_one, insert_many in Document.insert #1491
+-  Fix reload(fields) affect changed fields #1371
 
 Changes in 0.15.0
 =================