From 6a4c342e45fbd556dc1938923b7bed17031c79c7 Mon Sep 17 00:00:00 2001
From: Dmitry Voronenkov <divomen@gmail.com>
Date: Tue, 18 Jun 2019 16:13:29 +0300
Subject: [PATCH 01/40] Supported updates of an array by negative index

---
 mongoengine/base/datastructures.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py
index fafc08b7..9307556f 100644
--- a/mongoengine/base/datastructures.py
+++ b/mongoengine/base/datastructures.py
@@ -108,6 +108,9 @@ class BaseList(list):
         super(BaseList, self).__init__(list_items)
 
     def __getitem__(self, key):
+        # change index to positive value because MongoDB does not support negative one
+        if isinstance(key, int) and key < 0:
+            key = len(self) + key
         value = super(BaseList, self).__getitem__(key)
 
         if isinstance(key, slice):

From c68e3e1238bf77997dbe65415935abe8528ffef8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Wed, 24 Jul 2019 21:37:16 +0200
Subject: [PATCH 02/40] Add test case for list update by negative index

---
 tests/document/instance.py | 49 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/tests/document/instance.py b/tests/document/instance.py
index d8841a40..d0193b60 100644
--- a/tests/document/instance.py
+++ b/tests/document/instance.py
@@ -39,10 +39,10 @@ from tests.utils import MongoDBTestCase, get_as_pymongo
 
 TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), "../fields/mongoengine.png")
 
-__all__ = ("InstanceTest",)
+__all__ = ("TestDocumentInstance",)
 
 
-class InstanceTest(MongoDBTestCase):
+class TestDocumentInstance(MongoDBTestCase):
     def setUp(self):
         class Job(EmbeddedDocument):
             name = StringField()
@@ -3599,6 +3599,51 @@ class InstanceTest(MongoDBTestCase):
             self.assertEqual(b._instance, a)
         self.assertEqual(idx, 2)
 
+    def test_updating_listfield_manipulate_list(self):
+        class Company(Document):
+            name = StringField()
+            employees = ListField(field=DictField())
+
+        Company.drop_collection()
+
+        comp = Company(name="BigBank", employees=[{"name": "John"}])
+        comp.save()
+        comp.employees.append({"name": "Bill"})
+        comp.save()
+
+        stored_comp = get_as_pymongo(comp)
+        self.assertEqual(
+            stored_comp,
+            {
+                "_id": comp.id,
+                "employees": [{"name": "John"}, {"name": "Bill"}],
+                "name": "BigBank",
+            },
+        )
+
+        comp = comp.reload()
+        comp.employees[0]["color"] = "red"
+        comp.employees[-1]["color"] = "blue"
+        comp.employees[-1].update({"size": "xl"})
+        comp.save()
+
+        assert len(comp.employees) == 2
+        assert comp.employees[0] == {"name": "John", "color": "red"}
+        assert comp.employees[1] == {"name": "Bill", "size": "xl", "color": "blue"}
+
+        stored_comp = get_as_pymongo(comp)
+        self.assertEqual(
+            stored_comp,
+            {
+                "_id": comp.id,
+                "employees": [
+                    {"name": "John", "color": "red"},
+                    {"size": "xl", "color": "blue", "name": "Bill"},
+                ],
+                "name": "BigBank",
+            },
+        )
+
     def test_falsey_pk(self):
         """Ensure that we can create and update a document with Falsey PK."""
 

From 8f288fe45875050ac39985dcf9bcc304b6c1f15e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Sun, 29 Sep 2019 22:48:46 +0200
Subject: [PATCH 03/40] add mongodb 4.0 to travis and docs

---
 .travis.yml | 10 ++++++----
 README.rst  |  4 ++--
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 54a6befd..21321841 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,7 +3,7 @@
 # with a very large number of jobs, hence we only test a subset of all the
 # combinations:
 # * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup,
-#   tested against Python v2.7, v3.5, v3.6, and PyPy.
+#   tested against Python v2.7, v3.5, v3.6, v3.7, PyPy and PyPy3.
 # * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo
 #   combination: MongoDB v3.4, PyMongo v3.4, Python v2.7.
 # * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8.
@@ -30,15 +30,16 @@ dist: xenial
 
 env:
   global:
+    - MONGODB_4_0=4.0.12
     - MONGODB_3_4=3.4.17
     - MONGODB_3_6=3.6.12
+    - PYMONGO_3_9=3.9
     - PYMONGO_3_6=3.6
     - PYMONGO_3_4=3.4
   matrix:
-    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_6}
+    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_9}
 
 matrix:
-
   # Finish the build as soon as one job fails
   fast_finish: true
 
@@ -47,7 +48,8 @@ matrix:
     env: MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_4}
   - python: 3.7
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
-
+  - python: 3.7
+    env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_9}
 
 install:
   # Install Mongo
diff --git a/README.rst b/README.rst
index 679980f8..82b32893 100644
--- a/README.rst
+++ b/README.rst
@@ -26,10 +26,10 @@ an `API reference <https://mongoengine-odm.readthedocs.io/apireference.html>`_.
 
 Supported MongoDB Versions
 ==========================
-MongoEngine is currently tested against MongoDB v3.4 and v3.6. Future versions
+MongoEngine is currently tested against MongoDB v3.4, v3.6 and v4.0. Future versions
 should be supported as well, but aren't actively tested at the moment. Make
 sure to open an issue or submit a pull request if you experience any problems
-with MongoDB version > 3.6.
+with MongoDB version > 4.0.
 
 Installation
 ============

From b61c8cd104975b5fb47681387abc443c8c8430b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Tue, 1 Oct 2019 22:17:19 +0200
Subject: [PATCH 04/40] fix tox envs

---
 tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index a1ae8444..a7921c61 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py35,pypy,pypy3}-{mg34,mg36}
+envlist = {py27,py35,pypy,pypy3}-{mg34,mg36, mg39}
 
 [testenv]
 commands =
@@ -8,5 +8,6 @@ deps =
     nose
     mg34: pymongo>=3.4,<3.5
     mg36: pymongo>=3.6,<3.7
+    mg39: pymongo>=3.9,<4.0
 setenv =
     PYTHON_EGG_CACHE = {envdir}/python-eggs

From b2053144241f8f579993eff2c4de3084f074535f Mon Sep 17 00:00:00 2001
From: Matt Simpson <mcsimps2@gmail.com>
Date: Tue, 10 Dec 2019 11:09:22 -0500
Subject: [PATCH 05/40] Add ability for dict keys to have . or $ in MongoDB >=
 3.6

Starting in MongoDB >= 3.6, it is valid for dictionary keys to have $ or . in them as long as they don't start with $.  Additional tests added.
---
 AUTHORS                         |  1 +
 mongoengine/fields.py           | 19 ++++++++++++++++++-
 tests/fields/test_dict_field.py | 21 ++++++++++++++++-----
 3 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index aa044bd2..1271a8d9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -253,3 +253,4 @@ that much better:
  * Gaurav Dadhania (https://github.com/GVRV)
  * Yurii Andrieiev (https://github.com/yandrieiev)
  * Filip Kucharczyk (https://github.com/Pacu2)
+ * Matthew Simpson (https://github.com/mcsimps2)
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index f8f527a3..c3d93740 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -41,6 +41,7 @@ from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
+from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
 from mongoengine.python_support import StringIO
 from mongoengine.queryset import DO_NOTHING
 from mongoengine.queryset.base import BaseQuerySet
@@ -1051,6 +1052,15 @@ def key_has_dot_or_dollar(d):
             return True
 
 
+def key_starts_with_dollar(d):
+    """Helper function to recursively determine if any key in a
+    dictionary starts with a dollar
+    """
+    for k, v in d.items():
+        if (k.startswith("$")) or (isinstance(v, dict) and key_starts_with_dollar(v)):
+            return True
+
+
 class DictField(ComplexBaseField):
     """A dictionary field that wraps a standard Python dictionary. This is
     similar to an embedded document, but the structure is not defined.
@@ -1077,11 +1087,18 @@ class DictField(ComplexBaseField):
         if key_not_string(value):
             msg = "Invalid dictionary key - documents must have only string keys"
             self.error(msg)
-        if key_has_dot_or_dollar(value):
+
+        curr_mongo_ver = get_mongodb_version()
+
+        if curr_mongo_ver < MONGODB_36 and key_has_dot_or_dollar(value):
             self.error(
                 'Invalid dictionary key name - keys may not contain "."'
                 ' or startswith "$" characters'
             )
+        elif curr_mongo_ver >= MONGODB_36 and key_starts_with_dollar(value):
+            self.error(
+                'Invalid dictionary key name - keys may not startswith "$" characters'
+            )
         super(DictField, self).validate(value)
 
     def lookup_member(self, member_name):
diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py
index e88128f9..44e628f6 100644
--- a/tests/fields/test_dict_field.py
+++ b/tests/fields/test_dict_field.py
@@ -3,6 +3,7 @@ import pytest
 
 from mongoengine import *
 from mongoengine.base import BaseDict
+from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
 
 from tests.utils import MongoDBTestCase, get_as_pymongo
 
@@ -43,11 +44,7 @@ class TestDictField(MongoDBTestCase):
         with pytest.raises(ValidationError):
             post.validate()
 
-        post.info = {"the.title": "test"}
-        with pytest.raises(ValidationError):
-            post.validate()
-
-        post.info = {"nested": {"the.title": "test"}}
+        post.info = {"$title.test": "test"}
         with pytest.raises(ValidationError):
             post.validate()
 
@@ -55,6 +52,20 @@ class TestDictField(MongoDBTestCase):
         with pytest.raises(ValidationError):
             post.validate()
 
+        post.info = {"nested": {"the.title": "test"}}
+        if get_mongodb_version() < MONGODB_36:
+            with pytest.raises(ValidationError):
+                post.validate()
+        else:
+            post.validate()
+
+        post.info = {"dollar_and_dot": {"te$st.test": "test"}}
+        if get_mongodb_version() < MONGODB_36:
+            with pytest.raises(ValidationError):
+                post.validate()
+        else:
+            post.validate()
+
         post.info = {"title": "test"}
         post.save()
 

From 3b099f936a02444b3bf02c7dcdf13b1f2fc3b895 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 13 Dec 2019 21:32:45 +0100
Subject: [PATCH 06/40] provide additional details on how inheritance works in
 doc

---
 docs/guide/defining-documents.rst | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst
index 9dcca88c..652c5cd9 100644
--- a/docs/guide/defining-documents.rst
+++ b/docs/guide/defining-documents.rst
@@ -744,7 +744,7 @@ Document inheritance
 
 To create a specialised type of a :class:`~mongoengine.Document` you have
 defined, you may subclass it and add any extra fields or methods you may need.
-As this is new class is not a direct subclass of
+As this new class is not a direct subclass of
 :class:`~mongoengine.Document`, it will not be stored in its own collection; it
 will use the same collection as its superclass uses. This allows for more
 convenient and efficient retrieval of related documents -- all you need do is
@@ -767,6 +767,27 @@ document.::
           Setting :attr:`allow_inheritance` to True should also be used in
           :class:`~mongoengine.EmbeddedDocument` class in case you need to subclass it
 
+When it comes to querying using :attr:`.objects()`, querying `Page.objects()` will query
+both `Page` and `DatedPage` whereas querying `DatedPage` will only query the `DatedPage` documents.
+Behind the scenes, MongoEngine deals with inheritance by adding a :attr:`_cls` attribute that contains
+the class name in every documents. When a document is loaded, MongoEngine checks
+it's :attr:`_cls` attribute and use that class to construct the instance.::
+
+    Page(title='a funky title').save()
+    DatedPage(title='another title', date=datetime.utcnow()).save()
+
+    print(Page.objects().count())         # 2
+    print(DatedPage.objects().count())    # 1
+
+    # print documents in their native form
+    # we remove 'id' to avoid polluting the output with unnecessary detail
+    qs = Page.objects.exclude('id').as_pymongo()
+    print(list(qs))
+    # [
+    #   {'_cls': u 'Page', 'title': 'a funky title'},
+    #   {'_cls': u 'Page.DatedPage', 'title': u 'another title', 'date': datetime.datetime(2019, 12, 13, 20, 16, 59, 993000)}
+    # ]
+
 Working with existing data
 --------------------------
 As MongoEngine no longer defaults to needing :attr:`_cls`, you can quickly and

From 280a73af3bea8e9232a5ebe761d451840f025135 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Sat, 14 Dec 2019 21:44:59 +0100
Subject: [PATCH 07/40] minor fix in doc of NULLIFY to improve #834

---
 docs/guide/defining-documents.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst
index 9dcca88c..82388d3d 100644
--- a/docs/guide/defining-documents.rst
+++ b/docs/guide/defining-documents.rst
@@ -352,7 +352,7 @@ Its value can take any of the following constants:
   Deletion is denied if there still exist references to the object being
   deleted.
 :const:`mongoengine.NULLIFY`
-  Any object's fields still referring to the object being deleted are removed
+  Any object's fields still referring to the object being deleted are set to None
   (using MongoDB's "unset" operation), effectively nullifying the relationship.
 :const:`mongoengine.CASCADE`
   Any object containing fields that are referring to the object being deleted

From 50882e5bb09b74faddfac3cb93afd278ea94ced2 Mon Sep 17 00:00:00 2001
From: Eric Timmons <etimmons@mit.edu>
Date: Wed, 16 Oct 2019 09:49:40 -0400
Subject: [PATCH 08/40] Add failing test

Test that __eq__ for EmbeddedDocuments with LazyReferenceFields works as
expected.
---
 tests/document/test_instance.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py
index 173e02f2..6ba6827e 100644
--- a/tests/document/test_instance.py
+++ b/tests/document/test_instance.py
@@ -3319,6 +3319,38 @@ class TestInstance(MongoDBTestCase):
         f1.ref  # Dereferences lazily
         assert f1 == f2
 
+    def test_embedded_document_equality_with_lazy_ref(self):
+        class Job(EmbeddedDocument):
+            boss = LazyReferenceField('Person')
+
+        class Person(Document):
+            job = EmbeddedDocumentField(Job)
+
+        Person.drop_collection()
+
+        boss = Person()
+        worker = Person(job=Job(boss=boss))
+        boss.save()
+        worker.save()
+
+        worker1 = Person.objects.get(id=worker.id)
+
+        # worker1.job should be equal to the job used originally to create the
+        # document.
+        self.assertEqual(worker1.job, worker.job)
+
+        # worker1.job should be equal to a newly created Job EmbeddedDocument
+        # using either the Boss object or his ID.
+        self.assertEqual(worker1.job, Job(boss=boss))
+        self.assertEqual(worker1.job, Job(boss=boss.id))
+
+        # The above equalities should also hold after worker1.job.boss has been
+        # fetch()ed.
+        worker1.job.boss.fetch()
+        self.assertEqual(worker1.job, worker.job)
+        self.assertEqual(worker1.job, Job(boss=boss))
+        self.assertEqual(worker1.job, Job(boss=boss.id))
+
     def test_dbref_equality(self):
         class Test2(Document):
             name = StringField()

From dc7b96a5691335e970b13fb30ef62426b126e2bd Mon Sep 17 00:00:00 2001
From: Eric Timmons <etimmons@mit.edu>
Date: Wed, 16 Oct 2019 09:50:47 -0400
Subject: [PATCH 09/40] Make python value for LazyReferenceFields be a DBRef

Previously, when reading a LazyReferenceField from the DB, it was stored
internally in the parent document's _data field as an ObjectId. However, this
meant that equality tests using an enclosing EmbeddedDocument would not return
True when the EmbeddedDocument being compared to contained a DBRef or Document
in _data. Enclosing Documents were largely unaffected because they look at the
primary key for equality (which EmbeddedDocuments lack).

This makes the internal Python representation of a LazyReferenceField (before
the LazyReference itself has been constructed) a DBRef, using code identical to
ReferenceField.
---
 mongoengine/fields.py           | 9 +++++++++
 tests/document/test_instance.py | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index f8f527a3..0c29d1bc 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -2502,6 +2502,15 @@ class LazyReferenceField(BaseField):
         else:
             return pk
 
+    def to_python(self, value):
+        """Convert a MongoDB-compatible type to a Python type."""
+        if not self.dbref and not isinstance(
+            value, (DBRef, Document, EmbeddedDocument)
+        ):
+            collection = self.document_type._get_collection_name()
+            value = DBRef(collection, self.document_type.id.to_python(value))
+        return value
+
     def validate(self, value):
         if isinstance(value, LazyReference):
             if value.collection != self.document_type._get_collection_name():
diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py
index 6ba6827e..07376b4b 100644
--- a/tests/document/test_instance.py
+++ b/tests/document/test_instance.py
@@ -3321,7 +3321,7 @@ class TestInstance(MongoDBTestCase):
 
     def test_embedded_document_equality_with_lazy_ref(self):
         class Job(EmbeddedDocument):
-            boss = LazyReferenceField('Person')
+            boss = LazyReferenceField("Person")
 
         class Person(Document):
             job = EmbeddedDocumentField(Job)

From 0d4e61d489a9264863cecdfed08fc9e67a74d03a Mon Sep 17 00:00:00 2001
From: Eric Timmons <etimmons@mit.edu>
Date: Wed, 16 Oct 2019 10:01:19 -0400
Subject: [PATCH 10/40] Add daewok to AUTHORS per contributing guidelines

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index aa044bd2..374e2f7f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -253,3 +253,4 @@ that much better:
  * Gaurav Dadhania (https://github.com/GVRV)
  * Yurii Andrieiev (https://github.com/yandrieiev)
  * Filip Kucharczyk (https://github.com/Pacu2)
+ * Eric Timmons (https://github.com/daewok)

From 68dc2925fbea13702fa23ced0afd786d77b2ca28 Mon Sep 17 00:00:00 2001
From: Eric Timmons <etimmons@mit.edu>
Date: Sun, 15 Dec 2019 12:08:04 -0500
Subject: [PATCH 11/40] Add LazyReferenceField with dbref=True to
 embedded_document equality test

---
 tests/document/test_instance.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py
index 07376b4b..b899684f 100644
--- a/tests/document/test_instance.py
+++ b/tests/document/test_instance.py
@@ -3322,6 +3322,7 @@ class TestInstance(MongoDBTestCase):
     def test_embedded_document_equality_with_lazy_ref(self):
         class Job(EmbeddedDocument):
             boss = LazyReferenceField("Person")
+            boss_dbref = LazyReferenceField("Person", dbref=True)
 
         class Person(Document):
             job = EmbeddedDocumentField(Job)
@@ -3329,7 +3330,7 @@ class TestInstance(MongoDBTestCase):
         Person.drop_collection()
 
         boss = Person()
-        worker = Person(job=Job(boss=boss))
+        worker = Person(job=Job(boss=boss, boss_dbref=boss))
         boss.save()
         worker.save()
 
@@ -3341,15 +3342,15 @@ class TestInstance(MongoDBTestCase):
 
         # worker1.job should be equal to a newly created Job EmbeddedDocument
         # using either the Boss object or his ID.
-        self.assertEqual(worker1.job, Job(boss=boss))
-        self.assertEqual(worker1.job, Job(boss=boss.id))
+        self.assertEqual(worker1.job, Job(boss=boss, boss_dbref=boss))
+        self.assertEqual(worker1.job, Job(boss=boss.id, boss_dbref=boss.id))
 
         # The above equalities should also hold after worker1.job.boss has been
         # fetch()ed.
         worker1.job.boss.fetch()
         self.assertEqual(worker1.job, worker.job)
-        self.assertEqual(worker1.job, Job(boss=boss))
-        self.assertEqual(worker1.job, Job(boss=boss.id))
+        self.assertEqual(worker1.job, Job(boss=boss, boss_dbref=boss))
+        self.assertEqual(worker1.job, Job(boss=boss.id, boss_dbref=boss.id))
 
     def test_dbref_equality(self):
         class Test2(Document):

From 329f030a41da4d93aaec1f3ccee634e898f7d289 Mon Sep 17 00:00:00 2001
From: Eric Timmons <etimmons@mit.edu>
Date: Sun, 15 Dec 2019 20:15:13 -0500
Subject: [PATCH 12/40] Always store a DBRef, Document, or EmbeddedDocument in
 LazyReferenceField._data

This is required to handle the case of equality tests on a LazyReferenceField
with dbref=True when comparing against a field instantiated with an ObjectId.
---
 mongoengine/fields.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 0c29d1bc..a385559d 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -2504,9 +2504,7 @@ class LazyReferenceField(BaseField):
 
     def to_python(self, value):
         """Convert a MongoDB-compatible type to a Python type."""
-        if not self.dbref and not isinstance(
-            value, (DBRef, Document, EmbeddedDocument)
-        ):
+        if not isinstance(value, (DBRef, Document, EmbeddedDocument)):
             collection = self.document_type._get_collection_name()
             value = DBRef(collection, self.document_type.id.to_python(value))
         return value

From cfd4d6a161556ef4a8aa355468384554eb684442 Mon Sep 17 00:00:00 2001
From: Eric Timmons <etimmons@mit.edu>
Date: Sun, 15 Dec 2019 12:02:24 -0500
Subject: [PATCH 13/40] Add breaking change to changelog for LazyReferenceField
 representation in _data

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index bc01a403..b308c5fb 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -17,6 +17,7 @@ Development
     - If you catch/use ``MongoEngineConnectionError`` in your code, you'll have to rename it.
 - BREAKING CHANGE: Positional arguments when instantiating a document are no longer supported. #2103
     - From now on keyword arguments (e.g. ``Doc(field_name=value)``) are required.
+- BREAKING CHANGE: A ``LazyReferenceField`` is now stored in the ``_data`` field of its parent as a ``DBRef``, ``Document``, or ``EmbeddedDocument`` (``ObjectId`` is no longer allowed). #2182
 - DEPRECATION: ``Q.empty`` & ``QNode.empty`` are marked as deprecated and will be removed in a next version of MongoEngine. #2210
     - Added ability to check if Q or QNode are empty by parsing them to bool.
     - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``.

From ae326678ec0d7345645120c864b8b15cac741dc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 20 Dec 2019 22:09:04 +0100
Subject: [PATCH 14/40] updated changelog for upcoming 0.19.0

---
 docs/changelog.rst | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index b308c5fb..5c1e838a 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,8 +6,9 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
-- Documentation improvements:
-    - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver.
+
+Changes in 0.19.0
+=================
 - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112
     - Instead of ``Doc.objects(foo=bar, read_preference=...)`` use ``Doc.objects(foo=bar).read_preference(...)``.
     - Instead of ``Doc.objects(foo=bar, class_check=False)`` use ``Doc.objects(foo=bar).clear_cls_query(...)``.
@@ -21,14 +22,16 @@ Development
 - DEPRECATION: ``Q.empty`` & ``QNode.empty`` are marked as deprecated and will be removed in a next version of MongoEngine. #2210
     - Added ability to check if Q or QNode are empty by parsing them to bool.
     - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``.
-- Improve error message related to InvalidDocumentError #2180
 - Fix updating/modifying/deleting/reloading a document that's sharded by a field with ``db_field`` specified. #2125
 - ``ListField`` now accepts an optional ``max_length`` parameter. #2110
-- Switch from nosetest to pytest as test runner #2114
-- The codebase is now formatted using ``black``. #2109
-- In bulk write insert, the detailed error message would raise in exception.
+- Improve error message related to InvalidDocumentError #2180
+- Added BulkWriteError to replace NotUniqueError which was misleading in bulk write insert #2152
 - Added ability to compare Q and Q operations #2204
 - Added ability to use a db alias on query_counter #2194
+- Switch from nosetest to pytest as test runner #2114
+- The codebase is now formatted using ``black``. #2109
+- Documentation improvements:
+    - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver.
 
 Changes in 0.18.2
 =================

From 12d8bd5a22f0cee1f99f87b37bbaec94f0a002bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 20 Dec 2019 22:11:43 +0100
Subject: [PATCH 15/40] bump version to 0.19.0

---
 mongoengine/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py
index d7093d28..c41b5e70 100644
--- a/mongoengine/__init__.py
+++ b/mongoengine/__init__.py
@@ -28,7 +28,7 @@ __all__ = (
 )
 
 
-VERSION = (0, 18, 2)
+VERSION = (0, 19, 0)
 
 
 def get_version():

From d44533d95651559a65ce72664dca9ede98aa58e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 20 Dec 2019 22:41:22 +0100
Subject: [PATCH 16/40] completed the changelog with missing details

---
 docs/changelog.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 5c1e838a..93776a70 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -23,11 +23,13 @@ Changes in 0.19.0
     - Added ability to check if Q or QNode are empty by parsing them to bool.
     - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``.
 - Fix updating/modifying/deleting/reloading a document that's sharded by a field with ``db_field`` specified. #2125
+- Only set no_cursor_timeout when requested (fixes an incompatibility with MongoDB 4.2) #2148
 - ``ListField`` now accepts an optional ``max_length`` parameter. #2110
 - Improve error message related to InvalidDocumentError #2180
 - Added BulkWriteError to replace NotUniqueError which was misleading in bulk write insert #2152
 - Added ability to compare Q and Q operations #2204
 - Added ability to use a db alias on query_counter #2194
+- Added ability to specify collations for querysets with ``Doc.objects.collation`` #2024
 - Switch from nosetest to pytest as test runner #2114
 - The codebase is now formatted using ``black``. #2109
 - Documentation improvements:

From 332bd767d43af14bbb783e779d585cd2dbcf21de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 20 Dec 2019 22:51:08 +0100
Subject: [PATCH 17/40] minor fixes in tests

---
 docs/guide/mongomock.rst        |  6 +++---
 tests/document/test_instance.py | 22 +++++++++++-----------
 tests/fields/test_file_field.py |  2 +-
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/docs/guide/mongomock.rst b/docs/guide/mongomock.rst
index d70ee6a6..040ff912 100644
--- a/docs/guide/mongomock.rst
+++ b/docs/guide/mongomock.rst
@@ -2,10 +2,10 @@
 Use mongomock for testing
 ==============================
 
-`mongomock <https://github.com/vmalloc/mongomock/>`_ is a package to do just 
+`mongomock <https://github.com/vmalloc/mongomock/>`_ is a package to do just
 what the name implies, mocking a mongo database.
 
-To use with mongoengine, simply specify mongomock when connecting with 
+To use with mongoengine, simply specify mongomock when connecting with
 mongoengine:
 
 .. code-block:: python
@@ -45,4 +45,4 @@ Example of test file:
             pers.save()
 
             fresh_pers = Person.objects().first()
-            self.assertEqual(fresh_pers.name, 'John')
+            assert fresh_pers.name ==  'John'
diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py
index b899684f..609d0690 100644
--- a/tests/document/test_instance.py
+++ b/tests/document/test_instance.py
@@ -3338,19 +3338,19 @@ class TestInstance(MongoDBTestCase):
 
         # worker1.job should be equal to the job used originally to create the
         # document.
-        self.assertEqual(worker1.job, worker.job)
+        assert worker1.job == worker.job
 
         # worker1.job should be equal to a newly created Job EmbeddedDocument
         # using either the Boss object or his ID.
-        self.assertEqual(worker1.job, Job(boss=boss, boss_dbref=boss))
-        self.assertEqual(worker1.job, Job(boss=boss.id, boss_dbref=boss.id))
+        assert worker1.job == Job(boss=boss, boss_dbref=boss)
+        assert worker1.job == Job(boss=boss.id, boss_dbref=boss.id)
 
         # The above equalities should also hold after worker1.job.boss has been
         # fetch()ed.
         worker1.job.boss.fetch()
-        self.assertEqual(worker1.job, worker.job)
-        self.assertEqual(worker1.job, Job(boss=boss, boss_dbref=boss))
-        self.assertEqual(worker1.job, Job(boss=boss.id, boss_dbref=boss.id))
+        assert worker1.job == worker.job
+        assert worker1.job == Job(boss=boss, boss_dbref=boss)
+        assert worker1.job == Job(boss=boss.id, boss_dbref=boss.id)
 
     def test_dbref_equality(self):
         class Test2(Document):
@@ -3693,13 +3693,13 @@ class TestInstance(MongoDBTestCase):
         value = u"I_should_be_a_dict"
         coll.insert_one({"light_saber": value})
 
-        with self.assertRaises(InvalidDocumentError) as cm:
+        with pytest.raises(InvalidDocumentError) as exc_info:
             list(Jedi.objects)
 
-        self.assertEqual(
-            str(cm.exception),
-            "Invalid data to create a `Jedi` instance.\nField 'light_saber' - The source SON object needs to be of type 'dict' but a '%s' was found"
-            % type(value),
+        assert str(
+            exc_info.value
+        ) == "Invalid data to create a `Jedi` instance.\nField 'light_saber' - The source SON object needs to be of type 'dict' but a '%s' was found" % type(
+            value
         )
 
 
diff --git a/tests/fields/test_file_field.py b/tests/fields/test_file_field.py
index bfc86511..b8ece1a9 100644
--- a/tests/fields/test_file_field.py
+++ b/tests/fields/test_file_field.py
@@ -151,7 +151,7 @@ class TestFileField(MongoDBTestCase):
         result = StreamFile.objects.first()
         assert streamfile == result
         assert result.the_file.read() == text + more_text
-        # self.assertEqual(result.the_file.content_type, content_type)
+        # assert result.the_file.content_type == content_type
         result.the_file.seek(0)
         assert result.the_file.tell() == 0
         assert result.the_file.read(len(text)) == text

From 1170de1e8e30b976895c1c92cca134089dc5b806 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 20 Dec 2019 23:16:29 +0100
Subject: [PATCH 18/40] added explicit doc for order_by #2117

---
 docs/guide/mongomock.rst |  2 +-
 docs/guide/querying.rst  | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/docs/guide/mongomock.rst b/docs/guide/mongomock.rst
index 040ff912..141d7b69 100644
--- a/docs/guide/mongomock.rst
+++ b/docs/guide/mongomock.rst
@@ -21,7 +21,7 @@ or with an alias:
     conn = get_connection('testdb')
 
 Example of test file:
---------
+---------------------
 .. code-block:: python
 
     import unittest
diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst
index d64c169c..121325ae 100644
--- a/docs/guide/querying.rst
+++ b/docs/guide/querying.rst
@@ -222,6 +222,18 @@ keyword argument::
 
 .. versionadded:: 0.4
 
+Sorting/Ordering results
+========================
+It is possible to order the results by 1 or more keys using :meth:`~mongoengine.queryset.QuerySet.order_by`.
+The order may be specified by prepending each of the keys by "+" or "-". Ascending order is assumed if there's no prefix.::
+
+    # Order by ascending date
+    blogs = BlogPost.objects().order_by('date')    # equivalent to .order_by('+date')
+
+    # Order by ascending date first, then descending title
+    blogs = BlogPost.objects().order_by('+date', '-title')
+
+
 Limiting and skipping results
 =============================
 Just as with traditional ORMs, you may limit the number of results returned or
@@ -585,7 +597,8 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`::
     ['database', 'mongodb']
 
 From MongoDB version 2.6, push operator supports $position value which allows
-to push values with index.
+to push values with index::
+
     >>> post = BlogPost(title="Test", tags=["mongo"])
     >>> post.save()
     >>> post.update(push__tags__0=["database", "code"])

From 8e892dccfe01e284a99928a6ee4fbb1e0c35519d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 20 Dec 2019 23:51:01 +0100
Subject: [PATCH 19/40] document recent merged PR in changelog

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 93776a70..06eb8d0c 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -30,6 +30,7 @@ Changes in 0.19.0
 - Added ability to compare Q and Q operations #2204
 - Added ability to use a db alias on query_counter #2194
 - Added ability to specify collations for querysets with ``Doc.objects.collation`` #2024
+- Fix updates of a list field by negative index #2094
 - Switch from nosetest to pytest as test runner #2114
 - The codebase is now formatted using ``black``. #2109
 - Documentation improvements:

From 488604ff2e6cdadeed538de0dfb9fe36e27473b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Tue, 24 Dec 2019 00:00:15 +0100
Subject: [PATCH 20/40] test python 3.8

---
 .travis.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index cbf34cde..c825571b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,6 +23,7 @@ python:
 - 3.5
 - 3.6
 - 3.7
+- 3.8
 - pypy
 - pypy3
 
@@ -50,7 +51,8 @@ matrix:
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
   - python: 3.7
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
-
+  - python: 3.8
+    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
 
 install:
   # Install Mongo

From 62c8597a3b33dfd34cc785687c747606a0bbba9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Tue, 24 Dec 2019 11:15:23 +0100
Subject: [PATCH 21/40] fix pypi deployment version match

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index cbf34cde..6680a7e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -110,5 +110,5 @@ deploy:
   on:
     tags: true
     repo: MongoEngine/mongoengine
-    condition: ($PYMONGO = ${PYMONGO_3_6}) && ($MONGODB = ${MONGODB_3_4})
+    condition: ($PYMONGO = ${PYMONGO_3_9}) && ($MONGODB = ${MONGODB_3_4})
     python: 2.7

From ca4967311d3328fcc7abb84f8e9cdda7409e8dd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Thu, 26 Dec 2019 21:00:34 +0100
Subject: [PATCH 22/40] update python 3.8 config

---
 .travis.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 971a51a2..42f2a112 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -51,8 +51,6 @@ matrix:
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
   - python: 3.7
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
-  - python: 3.8
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
 
 install:
   # Install Mongo

From f0d1ee2cb41f1c9794eeee4de445fb27a7a77f88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Thu, 26 Dec 2019 21:03:34 +0100
Subject: [PATCH 23/40] update travis config + improve readmecode

---
 .travis.yml | 2 +-
 README.rst  | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 42f2a112..809fbad8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,7 +3,7 @@
 # with a very large number of jobs, hence we only test a subset of all the
 # combinations:
 # * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup,
-#   tested against Python v2.7, v3.5, v3.6, and PyPy.
+#   tested against Python v2.7, v3.5, v3.6, v3.7, v3.8 and PyPy.
 # * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo
 #   combination: MongoDB v3.4, PyMongo v3.4, Python v2.7.
 # * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8.
diff --git a/README.rst b/README.rst
index 853d8fbe..3b85fd48 100644
--- a/README.rst
+++ b/README.rst
@@ -91,12 +91,11 @@ Some simple examples of what MongoEngine code looks like:
 
     # Iterate over all posts using the BlogPost superclass
     >>> for post in BlogPost.objects:
-    ...     print '===', post.title, '==='
+    ...     print('===', post.title, '===')
     ...     if isinstance(post, TextPost):
-    ...         print post.content
+    ...         print(post.content)
     ...     elif isinstance(post, LinkPost):
-    ...         print 'Link:', post.url
-    ...     print
+    ...         print('Link:', post.url)
     ...
 
     # Count all blog posts and its subtypes

From aa02f87b69787ac678d4ee740e1cb2e5e6753fc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 27 Dec 2019 09:23:15 +0100
Subject: [PATCH 24/40] change & deprecate .aggregate api to mimic pymongo's
 interface + separate the aggregation tests from the large test_queryset.py
 file

---
 docs/guide/querying.rst                     |   4 +-
 mongoengine/queryset/base.py                |  29 ++-
 tests/queryset/test_queryset.py             | 236 +-----------------
 tests/queryset/test_queryset_aggregation.py | 255 ++++++++++++++++++++
 4 files changed, 277 insertions(+), 247 deletions(-)
 create mode 100644 tests/queryset/test_queryset_aggregation.py

diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst
index 121325ae..07de0378 100644
--- a/docs/guide/querying.rst
+++ b/docs/guide/querying.rst
@@ -400,7 +400,7 @@ would be generating "tag-clouds"::
 
 MongoDB aggregation API
 -----------------------
-If you need to run aggregation pipelines, MongoEngine provides an entry point `Pymongo's aggregation framework <https://api.mongodb.com/python/current/examples/aggregation.html#aggregation-framework>`_
+If you need to run aggregation pipelines, MongoEngine provides an entry point to `Pymongo's aggregation framework <https://api.mongodb.com/python/current/examples/aggregation.html#aggregation-framework>`_
 through :meth:`~mongoengine.queryset.QuerySet.aggregate`. Check out Pymongo's documentation for the syntax and pipeline.
 An example of its use would be::
 
@@ -414,7 +414,7 @@ An example of its use would be::
             {"$sort" : {"name" : -1}},
             {"$project": {"_id": 0, "name": {"$toUpper": "$name"}}}
             ]
-        data = Person.objects().aggregate(*pipeline)
+        data = Person.objects().aggregate(pipeline)
         assert data == [{'name': 'BOB'}, {'name': 'JOHN'}]
 
 Query efficiency and performance
diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index a648391e..aa5f2584 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -1255,16 +1255,25 @@ class BaseQuerySet(object):
             for data in son_data
         ]
 
-    def aggregate(self, *pipeline, **kwargs):
-        """
-        Perform a aggregate function based in your queryset params
+    def aggregate(self, pipeline, *suppl_pipeline, **kwargs):
+        """Perform a aggregate function based in your queryset params
+
         :param pipeline: list of aggregation commands,\
             see: http://docs.mongodb.org/manual/core/aggregation-pipeline/
-
+        :param suppl_pipeline: unpacked list of pipeline (added to support deprecation of the old interface)
+            parameter will be removed shortly
         .. versionadded:: 0.9
         """
-        initial_pipeline = []
+        using_deprecated_interface = isinstance(pipeline, dict) or bool(suppl_pipeline)
+        user_pipeline = [pipeline] if isinstance(pipeline, dict) else list(pipeline)
 
+        if using_deprecated_interface:
+            msg = "Calling .aggregate() with un unpacked list (*pipeline) is deprecated, it will soon change and will expect a list (similar to pymongo.Collection.aggregate interface), see documentation"
+            warnings.warn(msg, DeprecationWarning)
+
+        user_pipeline += suppl_pipeline
+
+        initial_pipeline = []
         if self._query:
             initial_pipeline.append({"$match": self._query})
 
@@ -1281,14 +1290,14 @@ class BaseQuerySet(object):
         if self._skip is not None:
             initial_pipeline.append({"$skip": self._skip})
 
-        pipeline = initial_pipeline + list(pipeline)
+        final_pipeline = initial_pipeline + user_pipeline
 
+        collection = self._collection
         if self._read_preference is not None:
-            return self._collection.with_options(
+            collection = self._collection.with_options(
                 read_preference=self._read_preference
-            ).aggregate(pipeline, cursor={}, **kwargs)
-
-        return self._collection.aggregate(pipeline, cursor={}, **kwargs)
+            )
+        return collection.aggregate(final_pipeline, cursor={}, **kwargs)
 
     # JS functionality
     def map_reduce(
diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py
index 7812ab66..b30350e6 100644
--- a/tests/queryset/test_queryset.py
+++ b/tests/queryset/test_queryset.py
@@ -14,7 +14,7 @@ import six
 from six import iteritems
 
 from mongoengine import *
-from mongoengine.connection import get_connection, get_db
+from mongoengine.connection import get_db
 from mongoengine.context_managers import query_counter, switch_db
 from mongoengine.errors import InvalidQueryError
 from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
@@ -4658,21 +4658,6 @@ class TestQueryset(unittest.TestCase):
         )
         assert_read_pref(bars, ReadPreference.SECONDARY_PREFERRED)
 
-    def test_read_preference_aggregation_framework(self):
-        class Bar(Document):
-            txt = StringField()
-
-            meta = {"indexes": ["txt"]}
-
-        # Aggregates with read_preference
-        bars = Bar.objects.read_preference(
-            ReadPreference.SECONDARY_PREFERRED
-        ).aggregate()
-        assert (
-            bars._CommandCursor__collection.read_preference
-            == ReadPreference.SECONDARY_PREFERRED
-        )
-
     def test_json_simple(self):
         class Embedded(EmbeddedDocument):
             string = StringField()
@@ -5399,225 +5384,6 @@ class TestQueryset(unittest.TestCase):
             assert Person.objects.first().name == "A"
             assert Person.objects._has_data(), "Cursor has data and returned False"
 
-    def test_queryset_aggregation_framework(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = Person.objects(age__lte=22).aggregate(
-            {"$project": {"name": {"$toUpper": "$name"}}}
-        )
-
-        assert list(data) == [
-            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
-            {"_id": p2.pk, "name": "WILSON JUNIOR"},
-        ]
-
-        data = (
-            Person.objects(age__lte=22)
-            .order_by("-name")
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [
-            {"_id": p2.pk, "name": "WILSON JUNIOR"},
-            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
-        ]
-
-        data = (
-            Person.objects(age__gte=17, age__lte=40)
-            .order_by("-age")
-            .aggregate(
-                {"$group": {"_id": None, "total": {"$sum": 1}, "avg": {"$avg": "$age"}}}
-            )
-        )
-        assert list(data) == [{"_id": None, "avg": 29, "total": 2}]
-
-        data = Person.objects().aggregate({"$match": {"name": "Isabella Luanna"}})
-        assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}]
-
-    def test_queryset_aggregation_with_skip(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = Person.objects.skip(1).aggregate(
-            {"$project": {"name": {"$toUpper": "$name"}}}
-        )
-
-        assert list(data) == [
-            {"_id": p2.pk, "name": "WILSON JUNIOR"},
-            {"_id": p3.pk, "name": "SANDRA MARA"},
-        ]
-
-    def test_queryset_aggregation_with_limit(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = Person.objects.limit(1).aggregate(
-            {"$project": {"name": {"$toUpper": "$name"}}}
-        )
-
-        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
-
-    def test_queryset_aggregation_with_sort(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = Person.objects.order_by("name").aggregate(
-            {"$project": {"name": {"$toUpper": "$name"}}}
-        )
-
-        assert list(data) == [
-            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
-            {"_id": p3.pk, "name": "SANDRA MARA"},
-            {"_id": p2.pk, "name": "WILSON JUNIOR"},
-        ]
-
-    def test_queryset_aggregation_with_skip_with_limit(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = list(
-            Person.objects.skip(1)
-            .limit(1)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}]
-
-        # Make sure limit/skip chaining order has no impact
-        data2 = (
-            Person.objects.limit(1)
-            .skip(1)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert data == list(data2)
-
-    def test_queryset_aggregation_with_sort_with_limit(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = (
-            Person.objects.order_by("name")
-            .limit(2)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [
-            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
-            {"_id": p3.pk, "name": "SANDRA MARA"},
-        ]
-
-        # Verify adding limit/skip steps works as expected
-        data = (
-            Person.objects.order_by("name")
-            .limit(2)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1})
-        )
-
-        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
-
-        data = (
-            Person.objects.order_by("name")
-            .limit(2)
-            .aggregate(
-                {"$project": {"name": {"$toUpper": "$name"}}},
-                {"$skip": 1},
-                {"$limit": 1},
-            )
-        )
-
-        assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}]
-
-    def test_queryset_aggregation_with_sort_with_skip(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = (
-            Person.objects.order_by("name")
-            .skip(2)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}]
-
-    def test_queryset_aggregation_with_sort_with_skip_with_limit(self):
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-
-        Person.drop_collection()
-
-        p1 = Person(name="Isabella Luanna", age=16)
-        p2 = Person(name="Wilson Junior", age=21)
-        p3 = Person(name="Sandra Mara", age=37)
-        Person.objects.insert([p1, p2, p3])
-
-        data = (
-            Person.objects.order_by("name")
-            .skip(1)
-            .limit(1)
-            .aggregate({"$project": {"name": {"$toUpper": "$name"}}})
-        )
-
-        assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}]
-
     def test_delete_count(self):
         [self.Person(name="User {0}".format(i), age=i * 10).save() for i in range(1, 4)]
         assert (
diff --git a/tests/queryset/test_queryset_aggregation.py b/tests/queryset/test_queryset_aggregation.py
new file mode 100644
index 00000000..00e04a36
--- /dev/null
+++ b/tests/queryset/test_queryset_aggregation.py
@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+import warnings
+
+from pymongo.read_preferences import ReadPreference
+
+from mongoengine import *
+from tests.utils import MongoDBTestCase
+
+
+class TestQuerysetAggregate(MongoDBTestCase):
+    def test_read_preference_aggregation_framework(self):
+        class Bar(Document):
+            txt = StringField()
+
+            meta = {"indexes": ["txt"]}
+
+        # Aggregates with read_preference
+        pipeline = []
+        bars = Bar.objects.read_preference(
+            ReadPreference.SECONDARY_PREFERRED
+        ).aggregate(pipeline)
+        assert (
+            bars._CommandCursor__collection.read_preference
+            == ReadPreference.SECONDARY_PREFERRED
+        )
+
+    def test_queryset_aggregation_framework(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects(age__lte=22).aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+            {"_id": p2.pk, "name": "WILSON JUNIOR"},
+        ]
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects(age__lte=22).order_by("-name").aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p2.pk, "name": "WILSON JUNIOR"},
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+        ]
+
+        pipeline = [
+            {"$group": {"_id": None, "total": {"$sum": 1}, "avg": {"$avg": "$age"}}}
+        ]
+        data = (
+            Person.objects(age__gte=17, age__lte=40)
+            .order_by("-age")
+            .aggregate(pipeline)
+        )
+        assert list(data) == [{"_id": None, "avg": 29, "total": 2}]
+
+        pipeline = [{"$match": {"name": "Isabella Luanna"}}]
+        data = Person.objects().aggregate(pipeline)
+        assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}]
+
+    def test_queryset_aggregation_with_skip(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.skip(1).aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p2.pk, "name": "WILSON JUNIOR"},
+            {"_id": p3.pk, "name": "SANDRA MARA"},
+        ]
+
+    def test_queryset_aggregation_with_limit(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.limit(1).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
+
+    def test_queryset_aggregation_with_sort(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+            {"_id": p3.pk, "name": "SANDRA MARA"},
+            {"_id": p2.pk, "name": "WILSON JUNIOR"},
+        ]
+
+    def test_queryset_aggregation_with_skip_with_limit(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = list(Person.objects.skip(1).limit(1).aggregate(pipeline))
+
+        assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}]
+
+        # Make sure limit/skip chaining order has no impact
+        data2 = Person.objects.limit(1).skip(1).aggregate(pipeline)
+
+        assert data == list(data2)
+
+    def test_queryset_aggregation_with_sort_with_limit(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").limit(2).aggregate(pipeline)
+
+        assert list(data) == [
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+            {"_id": p3.pk, "name": "SANDRA MARA"},
+        ]
+
+        # Verify adding limit/skip steps works as expected
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}]
+        data = Person.objects.order_by("name").limit(2).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
+
+        pipeline = [
+            {"$project": {"name": {"$toUpper": "$name"}}},
+            {"$skip": 1},
+            {"$limit": 1},
+        ]
+        data = Person.objects.order_by("name").limit(2).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}]
+
+    def test_queryset_aggregation_with_sort_with_skip(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").skip(2).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}]
+
+    def test_queryset_aggregation_with_sort_with_skip_with_limit(self):
+        class Person(Document):
+            name = StringField()
+            age = IntField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna", age=16)
+        p2 = Person(name="Wilson Junior", age=21)
+        p3 = Person(name="Sandra Mara", age=37)
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").skip(1).limit(1).aggregate(pipeline)
+
+        assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}]
+
+    def test_queryset_aggregation_deprecated_interface(self):
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+
+        p1 = Person(name="Isabella Luanna")
+        p2 = Person(name="Wilson Junior")
+        p3 = Person(name="Sandra Mara")
+        Person.objects.insert([p1, p2, p3])
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+
+        # Make sure a warning is emitted
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", DeprecationWarning)
+            with self.assertRaises(DeprecationWarning):
+                Person.objects.order_by("name").limit(2).aggregate(*pipeline)
+
+        # Make sure old interface works as expected with a 1-step pipeline
+        data = Person.objects.order_by("name").limit(2).aggregate(*pipeline)
+
+        assert list(data) == [
+            {"_id": p1.pk, "name": "ISABELLA LUANNA"},
+            {"_id": p3.pk, "name": "SANDRA MARA"},
+        ]
+
+        # Make sure old interface works as expected with a 2-steps pipeline
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}]
+        data = Person.objects.order_by("name").limit(2).aggregate(*pipeline)
+
+        assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
+
+
+if __name__ == "__main__":
+    unittest.main()

From 99e660c66d85620b427f2ab48eba557a6faa5a43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 27 Dec 2019 09:32:05 +0100
Subject: [PATCH 25/40] update changelog

---
 docs/changelog.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 06eb8d0c..5fe7d6b4 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,8 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
+- DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of
+    pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079
 
 Changes in 0.19.0
 =================

From b3dbb87c3c394cab5708bd321f58932d0c6b1063 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 27 Dec 2019 10:06:27 +0100
Subject: [PATCH 26/40] improve doc of aggregate kwargs

---
 mongoengine/queryset/base.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
index aa5f2584..50cb37ac 100644
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -302,7 +302,7 @@ class BaseQuerySet(object):
                 ``insert(..., {w: 2, fsync: True})`` will wait until at least
                 two servers have recorded the write and will force an fsync on
                 each server being written to.
-        :parm signal_kwargs: (optional) kwargs dictionary to be passed to
+        :param signal_kwargs: (optional) kwargs dictionary to be passed to
             the signal calls.
 
         By default returns document instances, set ``load_bulk`` to False to
@@ -1262,6 +1262,8 @@ class BaseQuerySet(object):
             see: http://docs.mongodb.org/manual/core/aggregation-pipeline/
         :param suppl_pipeline: unpacked list of pipeline (added to support deprecation of the old interface)
             parameter will be removed shortly
+        :param kwargs: (optional) kwargs dictionary to be passed to pymongo's aggregate call
+            See https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.aggregate
         .. versionadded:: 0.9
         """
         using_deprecated_interface = isinstance(pipeline, dict) or bool(suppl_pipeline)

From e7c7a66cd1d32fb1aa4595e3d8935c57de4beab1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 27 Dec 2019 10:38:03 +0100
Subject: [PATCH 27/40] improve doc of GridFS, emphasize that subsequent call
 to read() requires to rewind the file with seek(0)

---
 docs/guide/gridfs.rst | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst
index f7380e89..6e4a75a5 100644
--- a/docs/guide/gridfs.rst
+++ b/docs/guide/gridfs.rst
@@ -10,8 +10,9 @@ Writing
 GridFS support comes in the form of the :class:`~mongoengine.fields.FileField` field
 object. This field acts as a file-like object and provides a couple of
 different ways of inserting and retrieving data. Arbitrary metadata such as
-content type can also be stored alongside the files. In the following example,
-a document is created to store details about animals, including a photo::
+content type can also be stored alongside the files. The object returned when accessing a
+FileField is a proxy to `Pymongo's GridFS <https://api.mongodb.com/python/current/examples/gridfs.html#gridfs-example>`_
+In the following example, a document is created to store details about animals, including a photo::
 
     class Animal(Document):
         genus = StringField()
@@ -34,6 +35,20 @@ field. The file can also be retrieved just as easily::
     photo = marmot.photo.read()
     content_type = marmot.photo.content_type
 
+.. note:: If you need to read() the content of a file multiple times, you'll need to "rewind"
+    the file-like object using `seek`::
+
+        marmot = Animal.objects(genus='Marmota').first()
+        content1 = marmot.photo.read()
+        assert content1 != ""
+
+        content2 = marmot.photo.read()    # will be empty
+        assert content2 == ""
+
+        marmot.photo.seek(0)              # rewind the file by setting the current position of the cursor in the file to 0
+        content3 = marmot.photo.read()
+        assert content3 == content1
+
 Streaming
 ---------
 

From 152b51fd339a1bb9db254555461b5e6651bc3a6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Sun, 29 Dec 2019 14:36:50 +0100
Subject: [PATCH 28/40] improve gridfs example (properly opening file)

---
 docs/guide/gridfs.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst
index 6e4a75a5..0baf88e0 100644
--- a/docs/guide/gridfs.rst
+++ b/docs/guide/gridfs.rst
@@ -21,8 +21,8 @@ In the following example, a document is created to store details about animals,
 
     marmot = Animal(genus='Marmota', family='Sciuridae')
 
-    marmot_photo = open('marmot.jpg', 'rb')
-    marmot.photo.put(marmot_photo, content_type = 'image/jpeg')
+    with open('marmot.jpg', 'rb') as fd:
+        marmot.photo.put(fd, content_type = 'image/jpeg')
     marmot.save()
 
 Retrieval

From 4edad4601c7ccf30120cb8e45df7e1fb700a9347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 3 Jan 2020 14:03:17 +0100
Subject: [PATCH 29/40] Bump version to 0.19.1 + force pillow to be < 7.0.0

---
 docs/changelog.rst      | 4 ++++
 mongoengine/__init__.py | 2 +-
 setup.py                | 2 +-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 5fe7d6b4..b8e6ae56 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,10 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
+
+Changes in 0.19.1
+=================
+- Requires Pillow < 7.0.0 as it dropped Python2 support
 - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of
     pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079
 
diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py
index c41b5e70..e45dfc2b 100644
--- a/mongoengine/__init__.py
+++ b/mongoengine/__init__.py
@@ -28,7 +28,7 @@ __all__ = (
 )
 
 
-VERSION = (0, 19, 0)
+VERSION = (0, 19, 1)
 
 
 def get_version():
diff --git a/setup.py b/setup.py
index ceb5afad..2d69e44a 100644
--- a/setup.py
+++ b/setup.py
@@ -115,7 +115,7 @@ extra_opts = {
         "pytest-cov",
         "coverage<5.0",  # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls
         "blinker",
-        "Pillow>=2.0.0",
+        "Pillow>=2.0.0, <7.0.0",    # 7.0.0 dropped Python2 support
     ],
 }
 if sys.version_info[0] == 3:

From 75ee282a3dfc90ab2c6a4194e8f2851ce7f4543d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Fri, 3 Jan 2020 14:15:24 +0100
Subject: [PATCH 30/40] black setup.py to please CI

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2d69e44a..6c3ef8db 100644
--- a/setup.py
+++ b/setup.py
@@ -115,7 +115,7 @@ extra_opts = {
         "pytest-cov",
         "coverage<5.0",  # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls
         "blinker",
-        "Pillow>=2.0.0, <7.0.0",    # 7.0.0 dropped Python2 support
+        "Pillow>=2.0.0, <7.0.0",  # 7.0.0 dropped Python2 support
     ],
 }
 if sys.version_info[0] == 3:

From f8f267a880bcb9b17a51df61b006e7d9b92644b3 Mon Sep 17 00:00:00 2001
From: Tim Gates <tim.gates@iress.com>
Date: Sun, 5 Jan 2020 20:48:16 +1100
Subject: [PATCH 31/40] Fix simple typo: thorougly -> thoroughly

Closes #2236
---
 docs/upgrade.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/upgrade.rst b/docs/upgrade.rst
index 082dbadc..250347bf 100644
--- a/docs/upgrade.rst
+++ b/docs/upgrade.rst
@@ -52,7 +52,7 @@ rename its occurrences.
 This release includes a major rehaul of MongoEngine's code quality and
 introduces a few breaking changes. It also touches many different parts of
 the package and although all the changes have been tested and scrutinized,
-you're encouraged to thorougly test the upgrade.
+you're encouraged to thoroughly test the upgrade.
 
 First breaking change involves renaming `ConnectionError` to `MongoEngineConnectionError`.
 If you import or catch this exception, you'll need to rename it in your code.

From 59fbd505a04dd8cf822a098b834fdd8829395fc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Sun, 5 Jan 2020 20:20:13 +0100
Subject: [PATCH 32/40] include latest pymongo version in travis

---
 .travis.yml | 7 +++++--
 setup.py    | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 809fbad8..a299eea9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,11 +33,12 @@ env:
   global:
     - MONGODB_3_4=3.4.17
     - MONGODB_3_6=3.6.12
+    - PYMONGO_3_10=3.10
     - PYMONGO_3_9=3.9
     - PYMONGO_3_6=3.6
     - PYMONGO_3_4=3.4
   matrix:
-    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_9}
+    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10}
 
 matrix:
 
@@ -51,6 +52,8 @@ matrix:
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
   - python: 3.7
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
+  - python: 3.7
+    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_10}
 
 install:
   # Install Mongo
@@ -110,5 +113,5 @@ deploy:
   on:
     tags: true
     repo: MongoEngine/mongoengine
-    condition: ($PYMONGO = ${PYMONGO_3_9}) && ($MONGODB = ${MONGODB_3_4})
+    condition: ($PYMONGO = ${PYMONGO_3_10}) && ($MONGODB = ${MONGODB_3_4})
     python: 2.7
diff --git a/setup.py b/setup.py
index 6c3ef8db..5ba84e06 100644
--- a/setup.py
+++ b/setup.py
@@ -143,7 +143,7 @@ setup(
     long_description=LONG_DESCRIPTION,
     platforms=["any"],
     classifiers=CLASSIFIERS,
-    install_requires=["pymongo>=3.4", "six>=1.10.0"],
+    install_requires=["pymongo>=3.4, <4.0", "six>=1.10.0"],
     cmdclass={"test": PyTest},
     **extra_opts
 )

From 705c55ce24c006dd2bd06437f3d7fb945ed1de17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Sun, 5 Jan 2020 20:30:56 +0100
Subject: [PATCH 33/40] update  tox file to account for mg310

---
 tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 349b5577..a3d2df60 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,6 +7,7 @@ commands =
 deps =
     mg34: pymongo>=3.4,<3.5
     mg36: pymongo>=3.6,<3.7
-    mg39: pymongo>=3.9,<4.0
+    mg39: pymongo>=3.9,<3.10
+    mg310: pymongo>=3.10,<3.11
 setenv =
     PYTHON_EGG_CACHE = {envdir}/python-eggs

From 18b68f1b8034457384daa5bdbd7d780055e21690 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Sun, 12 Jan 2020 21:29:18 +0100
Subject: [PATCH 34/40] update travis mongo 4.0 to latest 4.0.13

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f7880649..a7d6da1a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,7 +31,7 @@ dist: xenial
 
 env:
   global:
-    - MONGODB_4_0=4.0.12
+    - MONGODB_4_0=4.0.13
     - MONGODB_3_4=3.4.17
     - MONGODB_3_6=3.6.12
     - PYMONGO_3_10=3.10

From e0565ddac5cd62f9fd05c69187b49a20bb644035 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Sun, 12 Jan 2020 21:31:28 +0100
Subject: [PATCH 35/40] update changelog

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index b8e6ae56..d924a2c1 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,7 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
+- Add Mongo 4.0 to Travis
 
 Changes in 0.19.1
 =================

From 605de59bd08d9d4f54f695c7f73a1458975d479f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Sun, 12 Jan 2020 21:37:32 +0100
Subject: [PATCH 36/40] improve travis + fix tox mg310

---
 .travis.yml | 9 +++++----
 tox.ini     | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a7d6da1a..62bbacb1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,13 +31,14 @@ dist: xenial
 
 env:
   global:
-    - MONGODB_4_0=4.0.13
     - MONGODB_3_4=3.4.17
     - MONGODB_3_6=3.6.12
-    - PYMONGO_3_10=3.10
-    - PYMONGO_3_9=3.9
-    - PYMONGO_3_6=3.6
+    - MONGODB_4_0=4.0.13
+
     - PYMONGO_3_4=3.4
+    - PYMONGO_3_6=3.6
+    - PYMONGO_3_9=3.9
+    - PYMONGO_3_10=3.10
   matrix:
     - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10}
 
diff --git a/tox.ini b/tox.ini
index c3789b7d..396817ca 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py35,pypy,pypy3}-{mg34,mg36, mg39}
+envlist = {py27,py35,pypy,pypy3}-{mg34,mg36,mg39,mg310}
 
 [testenv]
 commands =

From 450658d7ac2b26be7849f8f0599fff06bb7e6ebc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastien=20G=C3=A9rard?= <bastien.gerard@cluepoints.com>
Date: Tue, 4 Feb 2020 22:51:02 +0100
Subject: [PATCH 37/40] fix indirect library version that dropped python2
 support recently

---
 setup.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 5ba84e06..5cba5d9e 100644
--- a/setup.py
+++ b/setup.py
@@ -108,6 +108,10 @@ CLASSIFIERS = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 
+PYTHON_VERSION = sys.version_info[0]
+PY3 = PYTHON_VERSION == 3
+PY2 = PYTHON_VERSION == 2
+
 extra_opts = {
     "packages": find_packages(exclude=["tests", "tests.*"]),
     "tests_require": [
@@ -116,9 +120,10 @@ extra_opts = {
         "coverage<5.0",  # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls
         "blinker",
         "Pillow>=2.0.0, <7.0.0",  # 7.0.0 dropped Python2 support
+        "zipp<2.0.0",  # (dependency of pytest) dropped python2 support
     ],
 }
-if sys.version_info[0] == 3:
+if PY3:
     extra_opts["use_2to3"] = True
     if "test" in sys.argv:
         extra_opts["packages"] = find_packages()

From 81f9b351b3838fe27924937e706300805e9eb82b Mon Sep 17 00:00:00 2001
From: Leonardo Domingues <leodmgs@gmail.com>
Date: Fri, 21 Feb 2020 19:14:34 -0300
Subject: [PATCH 38/40] Add return info in the save function docstring

---
 AUTHORS                 | 1 +
 mongoengine/document.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index b9a81c63..7d3000ce 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -255,3 +255,4 @@ that much better:
  * Filip Kucharczyk (https://github.com/Pacu2)
  * Eric Timmons (https://github.com/daewok)
  * Matthew Simpson (https://github.com/mcsimps2)
+ * Leonardo Domingues (https://github.com/leodmgs)
diff --git a/mongoengine/document.py b/mongoengine/document.py
index 23968f17..5e812510 100644
--- a/mongoengine/document.py
+++ b/mongoengine/document.py
@@ -332,7 +332,7 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)):
     ):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
-        created.
+        created. Returns the saved object instance.
 
         :param force_insert: only try to create a new document, don't allow
             updates of existing documents.

From d287f480e5016090384650515c9342c767c4bbb7 Mon Sep 17 00:00:00 2001
From: Filip Kucharczyk <filip.m.kucharczyk@gmail.com>
Date: Tue, 4 Feb 2020 12:35:03 +0100
Subject: [PATCH 39/40] Fix for combining raw and regular filters

---
 mongoengine/queryset/transform.py | 4 ++--
 tests/queryset/test_transform.py  | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py
index 0b73e99b..659a97e2 100644
--- a/mongoengine/queryset/transform.py
+++ b/mongoengine/queryset/transform.py
@@ -169,9 +169,9 @@ def query(_doc_cls=None, **kwargs):
 
         key = ".".join(parts)
 
-        if op is None or key not in mongo_query:
+        if key not in mongo_query:
             mongo_query[key] = value
-        elif key in mongo_query:
+        else:
             if isinstance(mongo_query[key], dict) and isinstance(value, dict):
                 mongo_query[key].update(value)
                 # $max/minDistance needs to come last - convert to SON
diff --git a/tests/queryset/test_transform.py b/tests/queryset/test_transform.py
index 3898809e..8d6c2d06 100644
--- a/tests/queryset/test_transform.py
+++ b/tests/queryset/test_transform.py
@@ -24,6 +24,12 @@ class TestTransform(unittest.TestCase):
         }
         assert transform.query(friend__age__gte=30) == {"friend.age": {"$gte": 30}}
         assert transform.query(name__exists=True) == {"name": {"$exists": True}}
+        assert transform.query(name=["Mark"], __raw__={"name": {"$in": "Tom"}}) == {
+            "$and": [{"name": ["Mark"]}, {"name": {"$in": "Tom"}}]
+        }
+        assert transform.query(name__in=["Tom"], __raw__={"name": "Mark"}) == {
+            "$and": [{"name": {"$in": ["Tom"]}}, {"name": "Mark"}]
+        }
 
     def test_transform_update(self):
         class LisDoc(Document):

From fda2e2b47ab5085b6ff5b6e27b40794a3fa9e77f Mon Sep 17 00:00:00 2001
From: Filip Kucharczyk <filip.m.kucharczyk@gmail.com>
Date: Tue, 4 Feb 2020 12:58:25 +0100
Subject: [PATCH 40/40] Update changelog

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index d924a2c1..8dcea62a 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -7,6 +7,7 @@ Development
 ===========
 - (Fill this out as you fix issues and develop your features).
 - Add Mongo 4.0 to Travis
+- Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264
 
 Changes in 0.19.1
 =================